CORD-48 Implementation of hashing Next Objective in OF-DPA driver. Major changes…

… to ensure multi-ONOS-instance group-chain installation.
Also includes:
     Changes to Next Objective that adds metadata field for applications to optionally send auxillary info to drivers
     Changes to Next Objective that allows more explicit modification of the next objective
     Changes to Forwarding Objective and PendingNext to include hashCode() and equals() method
     MplsBosInstruction included in kryo serializer
     GroupKey's byte[] represented as a hex string
     Bug fix in mpls flow installation to report failure in install
     Bug fix in linkUp in SR app to disallow non-masters to modify groups
     Bug fix in ordering of actions in group

Change-Id: I3e7003f55724c2de79589e43e11d05ff4815a81d

@@ -448,7 +448,6 @@ public class DefaultRoutingHandler {
         if (nextHops.isEmpty()) {
         if (nextHops.isEmpty()) {
             nextHops.add(destSw);
             nextHops.add(destSw);
         }
         }
-
         // If both target switch and dest switch are edge routers, then set IP
         // If both target switch and dest switch are edge routers, then set IP
         // rule for both subnet and router IP.
         // rule for both subnet and router IP.
         boolean targetIsEdge;
         boolean targetIsEdge;
@@ -485,8 +484,8 @@ public class DefaultRoutingHandler {
                 return false;
                 return false;
             }
             }
 
 
-        // If the target switch is an edge router, then set IP rules for the router IP.
         } else if (targetIsEdge) {
         } else if (targetIsEdge) {
+            // If the target switch is an edge router, then set IP rules for the router IP.
             Ip4Address routerIp = destRouterIp;
             Ip4Address routerIp = destRouterIp;
             IpPrefix routerIpPrefix = IpPrefix.valueOf(routerIp, IpPrefix.MAX_INET_MASK_LENGTH);
             IpPrefix routerIpPrefix = IpPrefix.valueOf(routerIp, IpPrefix.MAX_INET_MASK_LENGTH);
             log.debug("* populateEcmpRoutingRulePartial in device {} towards {} for router IP {}",
             log.debug("* populateEcmpRoutingRulePartial in device {} towards {} for router IP {}",
@@ -496,7 +495,6 @@ public class DefaultRoutingHandler {
                 return false;
                 return false;
             }
             }
         }
         }
-
         // Populates MPLS rules to all routers
         // Populates MPLS rules to all routers
         log.debug("* populateEcmpRoutingRulePartial in device{} towards {} for all MPLS rules",
         log.debug("* populateEcmpRoutingRulePartial in device{} towards {} for all MPLS rules",
                 targetSw, destSw);
                 targetSw, destSw);
@@ -504,7 +502,6 @@ public class DefaultRoutingHandler {
         if (!result) {
         if (!result) {
             return false;
             return false;
         }
         }
-
         return true;
         return true;
     }
     }
 
 

@@ -15,6 +15,7 @@
  */
  */
 package org.onosproject.segmentrouting;
 package org.onosproject.segmentrouting;
 
 
+import org.onlab.packet.EthType;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.Ip4Prefix;
 import org.onlab.packet.Ip4Prefix;
@@ -26,7 +27,6 @@ import org.onosproject.segmentrouting.config.DeviceConfigNotFoundException;
 import org.onosproject.segmentrouting.config.DeviceConfiguration;
 import org.onosproject.segmentrouting.config.DeviceConfiguration;
 import org.onosproject.segmentrouting.grouphandler.NeighborSet;
 import org.onosproject.segmentrouting.grouphandler.NeighborSet;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.DeviceId;
-import org.onosproject.net.Link;
 import org.onosproject.net.Port;
 import org.onosproject.net.Port;
 import org.onosproject.net.PortNumber;
 import org.onosproject.net.PortNumber;
 import org.onosproject.net.flow.DefaultTrafficSelector;
 import org.onosproject.net.flow.DefaultTrafficSelector;
@@ -227,7 +227,15 @@ public class RoutingRulePopulator {
             treatment = null;
             treatment = null;
         }
         }
 
 
-        if (srManager.getNextObjectiveId(deviceId, ns) <= 0) {
+        // setup metadata to pass to nextObjective - indicate the vlan on egress
+        // if needed by the switch pipeline. Since neighbor sets are always to
+        // other neighboring routers, there is no subnet assigned on those ports.
+        TrafficSelector.Builder metabuilder = DefaultTrafficSelector.builder(selector);
+        metabuilder.matchVlanId(
+            VlanId.vlanId(SegmentRoutingManager.ASSIGNED_VLAN_NO_SUBNET));
+
+        int nextId = srManager.getNextObjectiveId(deviceId, ns, metabuilder.build());
+        if (nextId <= 0) {
             log.warn("No next objective in {} for ns: {}", deviceId, ns);
             log.warn("No next objective in {} for ns: {}", deviceId, ns);
             return false;
             return false;
         }
         }
@@ -236,7 +244,7 @@ public class RoutingRulePopulator {
                 .builder()
                 .builder()
                 .fromApp(srManager.appId)
                 .fromApp(srManager.appId)
                 .makePermanent()
                 .makePermanent()
-                .nextStep(srManager.getNextObjectiveId(deviceId, ns))
+                .nextStep(nextId)
                 .withSelector(selector)
                 .withSelector(selector)
                 .withPriority(100)
                 .withPriority(100)
                 .withFlag(ForwardingObjective.Flag.SPECIFIC);
                 .withFlag(ForwardingObjective.Flag.SPECIFIC);
@@ -279,63 +287,70 @@ public class RoutingRulePopulator {
         List<ForwardingObjective.Builder> fwdObjBuilders = new ArrayList<>();
         List<ForwardingObjective.Builder> fwdObjBuilders = new ArrayList<>();
 
 
         // TODO Handle the case of Bos == false
         // TODO Handle the case of Bos == false
-        sbuilder.matchMplsLabel(MplsLabel.mplsLabel(segmentId));
         sbuilder.matchEthType(Ethernet.MPLS_UNICAST);
         sbuilder.matchEthType(Ethernet.MPLS_UNICAST);
+        sbuilder.matchMplsLabel(MplsLabel.mplsLabel(segmentId));
+        TrafficSelector selector = sbuilder.build();
+
+        // setup metadata to pass to nextObjective - indicate the vlan on egress
+        // if needed by the switch pipeline. Since mpls next-hops are always to
+        // other neighboring routers, there is no subnet assigned on those ports.
+        TrafficSelector.Builder metabuilder = DefaultTrafficSelector.builder(selector);
+        metabuilder.matchVlanId(
+            VlanId.vlanId(SegmentRoutingManager.ASSIGNED_VLAN_NO_SUBNET));
 
 
-        // If the next hop is the destination router, do PHP
+        // If the next hop is the destination router for the segment, do pop
         if (nextHops.size() == 1 && destSwId.equals(nextHops.toArray()[0])) {
         if (nextHops.size() == 1 && destSwId.equals(nextHops.toArray()[0])) {
             log.debug("populateMplsRule: Installing MPLS forwarding objective for "
             log.debug("populateMplsRule: Installing MPLS forwarding objective for "
-                    + "label {} in switch {} with PHP",
-                    segmentId,
-                    deviceId);
+                    + "label {} in switch {} with pop", segmentId, deviceId);
 
 
+            // bos pop case (php)
             ForwardingObjective.Builder fwdObjBosBuilder =
             ForwardingObjective.Builder fwdObjBosBuilder =
                     getMplsForwardingObjective(deviceId,
                     getMplsForwardingObjective(deviceId,
-                                               destSwId,
                                                nextHops,
                                                nextHops,
                                                true,
                                                true,
-                                               true);
-            // TODO: Check with Sangho on why we need this
-            ForwardingObjective.Builder fwdObjNoBosBuilder =
-                    getMplsForwardingObjective(deviceId,
-                                               destSwId,
-                                               nextHops,
                                                true,
                                                true,
-                                               false);
-            if (fwdObjBosBuilder != null) {
-                fwdObjBuilders.add(fwdObjBosBuilder);
-            } else {
-                log.warn("Failed to set MPLS rules.");
+                                               metabuilder.build());
+            if (fwdObjBosBuilder == null) {
                 return false;
                 return false;
             }
             }
+            fwdObjBuilders.add(fwdObjBosBuilder);
+
+            // XXX not-bos pop case,  SR app multi-label not implemented yet
+            /*ForwardingObjective.Builder fwdObjNoBosBuilder =
+                    getMplsForwardingObjective(deviceId,
+                                               nextHops,
+                                               true,
+                                               false);*/
+
         } else {
         } else {
+            // next hop is not destination, SR CONTINUE case (swap with self)
             log.debug("Installing MPLS forwarding objective for "
             log.debug("Installing MPLS forwarding objective for "
-                    + "label {} in switch {} without PHP",
-                    segmentId,
-                    deviceId);
+                    + "label {} in switch {} without pop", segmentId, deviceId);
 
 
+            // continue case with bos - this does get triggered in edge routers
+            // and in core routers - driver can handle depending on availability
+            // of MPLS ECMP or not
             ForwardingObjective.Builder fwdObjBosBuilder =
             ForwardingObjective.Builder fwdObjBosBuilder =
                     getMplsForwardingObjective(deviceId,
                     getMplsForwardingObjective(deviceId,
-                                               destSwId,
                                                nextHops,
                                                nextHops,
                                                false,
                                                false,
-                                               true);
-            // TODO: Check with Sangho on why we need this
-            ForwardingObjective.Builder fwdObjNoBosBuilder =
+                                               true,
+                                               metabuilder.build());
+            if (fwdObjBosBuilder == null) {
+                return false;
+            }
+            fwdObjBuilders.add(fwdObjBosBuilder);
+
+            // XXX continue case with not-bos - SR app multi label not implemented yet
+            // also requires MPLS ECMP
+            /*ForwardingObjective.Builder fwdObjNoBosBuilder =
                     getMplsForwardingObjective(deviceId,
                     getMplsForwardingObjective(deviceId,
-                                               destSwId,
                                                nextHops,
                                                nextHops,
                                                false,
                                                false,
-                                               false);
-            if (fwdObjBosBuilder != null) {
-                fwdObjBuilders.add(fwdObjBosBuilder);
-            } else {
-                log.warn("Failed to set MPLS rules.");
-                return false;
-            }
+                                               false); */
+
         }
         }
 
 
-        TrafficSelector selector = sbuilder.build();
         for (ForwardingObjective.Builder fwdObjBuilder : fwdObjBuilders) {
         for (ForwardingObjective.Builder fwdObjBuilder : fwdObjBuilders) {
             ((Builder) ((Builder) fwdObjBuilder.fromApp(srManager.appId)
             ((Builder) ((Builder) fwdObjBuilder.fromApp(srManager.appId)
                     .makePermanent()).withSelector(selector)
                     .makePermanent()).withSelector(selector)
@@ -352,67 +367,52 @@ public class RoutingRulePopulator {
         return true;
         return true;
     }
     }
 
 
-    private ForwardingObjective.Builder getMplsForwardingObjective(DeviceId deviceId,
-                                                                   DeviceId destSw,
+    private ForwardingObjective.Builder getMplsForwardingObjective(
+                                             DeviceId deviceId,
                                              Set<DeviceId> nextHops,
                                              Set<DeviceId> nextHops,
                                              boolean phpRequired,
                                              boolean phpRequired,
-                                                                   boolean isBos) {
+                                             boolean isBos,
+                                             TrafficSelector meta) {
+
         ForwardingObjective.Builder fwdBuilder = DefaultForwardingObjective
         ForwardingObjective.Builder fwdBuilder = DefaultForwardingObjective
                 .builder().withFlag(ForwardingObjective.Flag.SPECIFIC);
                 .builder().withFlag(ForwardingObjective.Flag.SPECIFIC);
-        DeviceId nextHop = (DeviceId) nextHops.toArray()[0];
-
-        boolean isEdge;
-        MacAddress srcMac;
-        MacAddress dstMac;
-        try {
-            isEdge = config.isEdgeDevice(deviceId);
-            srcMac = config.getDeviceMac(deviceId);
-            dstMac = config.getDeviceMac(nextHop);
-        } catch (DeviceConfigNotFoundException e) {
-            log.warn(e.getMessage() + " Aborting getMplsForwardingObjective");
-            return null;
-        }
 
 
         TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
         TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
 
 
         if (phpRequired) {
         if (phpRequired) {
+            // php case - pop should always be flow-action
             log.debug("getMplsForwardingObjective: php required");
             log.debug("getMplsForwardingObjective: php required");
             tbuilder.deferred().copyTtlIn();
             tbuilder.deferred().copyTtlIn();
             if (isBos) {
             if (isBos) {
-                tbuilder.deferred().popMpls(Ethernet.TYPE_IPV4).decNwTtl();
+                tbuilder.deferred().popMpls(EthType.EtherType.IPV4.ethType())
+                    .decNwTtl();
             } else {
             } else {
-                tbuilder.deferred().popMpls(Ethernet.MPLS_UNICAST).decMplsTtl();
+                tbuilder.deferred().popMpls(EthType.EtherType.MPLS_UNICAST.ethType())
+                    .decMplsTtl();
             }
             }
         } else {
         } else {
+            // swap with self case - SR CONTINUE
             log.debug("getMplsForwardingObjective: php not required");
             log.debug("getMplsForwardingObjective: php not required");
             tbuilder.deferred().decMplsTtl();
             tbuilder.deferred().decMplsTtl();
         }
         }
 
 
-        if (!isECMPSupportedInTransitRouter() && !isEdge) {
-            PortNumber port = selectOnePort(deviceId, nextHops);
-            if (port == null) {
-                log.warn("No link from {} to {}", deviceId, nextHops);
-                return null;
-            }
-            tbuilder.deferred()
-                    .setEthSrc(srcMac)
-                    .setEthDst(dstMac)
-                    .setOutput(port);
+        // All forwarding is via ECMP group, the metadata informs the driver
+        // that the next-Objective will be used by MPLS flows. In other words,
+        // MPLS ECMP is requested. It is up to the driver to decide if these
+        // packets will be hashed or not.
         fwdBuilder.withTreatment(tbuilder.build());
         fwdBuilder.withTreatment(tbuilder.build());
-        } else {
         NeighborSet ns = new NeighborSet(nextHops);
         NeighborSet ns = new NeighborSet(nextHops);
-            fwdBuilder.withTreatment(tbuilder.build());
-            fwdBuilder.nextStep(srManager
-                    .getNextObjectiveId(deviceId, ns));
-        }
+        log.debug("Trying to get a nextObjid for mpls rule on device:{} to ns:{}",
+                 deviceId, ns);
 
 
-        return fwdBuilder;
+        int nextId = srManager.getNextObjectiveId(deviceId, ns, meta);
+        if (nextId <= 0) {
+            log.warn("No next objective in {} for ns: {}", deviceId, ns);
+            return null;
         }
         }
 
 
-    private boolean isECMPSupportedInTransitRouter() {
-
-        // TODO: remove this function when objectives subsystem is supported.
-        return false;
+        fwdBuilder.nextStep(nextId);
+        return fwdBuilder;
     }
     }
 
 
     /**
     /**
@@ -552,22 +552,6 @@ public class RoutingRulePopulator {
     }
     }
 
 
 
 
-    private PortNumber selectOnePort(DeviceId srcId, Set<DeviceId> destIds) {
-
-        Set<Link> links = srManager.linkService.getDeviceLinks(srcId);
-        for (DeviceId destId: destIds) {
-            for (Link link : links) {
-                if (link.dst().deviceId().equals(destId)) {
-                    return link.src().port();
-                } else if (link.src().deviceId().equals(destId)) {
-                    return link.dst().port();
-                }
-            }
-        }
-
-        return null;
-    }
-
     private static class SRObjectiveContext implements ObjectiveContext {
     private static class SRObjectiveContext implements ObjectiveContext {
         enum ObjectiveType {
         enum ObjectiveType {
             FILTER,
             FILTER,

@@ -424,17 +424,22 @@ public class SegmentRoutingManager implements SegmentRoutingService {
 
 
     /**
     /**
      * Returns the next objective ID for the given NeighborSet.
      * Returns the next objective ID for the given NeighborSet.
-     * If the nextObjectiveID does not exist, a new one is created and returned.
+     * If the nextObjective does not exist, a new one is created and
+     * it's id is returned.
+     * TODO move the side-effect creation of a Next Objective into a new method
      *
      *
      * @param deviceId Device ID
      * @param deviceId Device ID
      * @param ns NegighborSet
      * @param ns NegighborSet
-     * @return next objective ID
+     * @param meta metadata passed into the creation of a Next Objective
+     * @return next objective ID or -1 if an error was encountered during the
+     *         creation of the nextObjective
      */
      */
-    public int getNextObjectiveId(DeviceId deviceId, NeighborSet ns) {
+    public int getNextObjectiveId(DeviceId deviceId, NeighborSet ns,
+                                  TrafficSelector meta) {
         if (groupHandlerMap.get(deviceId) != null) {
         if (groupHandlerMap.get(deviceId) != null) {
             log.trace("getNextObjectiveId query in device {}", deviceId);
             log.trace("getNextObjectiveId query in device {}", deviceId);
             return groupHandlerMap
             return groupHandlerMap
-                    .get(deviceId).getNextObjectiveId(ns);
+                    .get(deviceId).getNextObjectiveId(ns, meta);
         } else {
         } else {
             log.warn("getNextObjectiveId query in device {} not found", deviceId);
             log.warn("getNextObjectiveId query in device {} not found", deviceId);
             return -1;
             return -1;
@@ -586,7 +591,8 @@ public class SegmentRoutingManager implements SegmentRoutingService {
         DefaultGroupHandler groupHandler = groupHandlerMap.get(link.src()
         DefaultGroupHandler groupHandler = groupHandlerMap.get(link.src()
                 .deviceId());
                 .deviceId());
         if (groupHandler != null) {
         if (groupHandler != null) {
-            groupHandler.linkUp(link);
+            groupHandler.linkUp(link, mastershipService.isLocalMaster(
+                                           link.src().deviceId()));
         } else {
         } else {
             Device device = deviceService.getDevice(link.src().deviceId());
             Device device = deviceService.getDevice(link.src().deviceId());
             if (device != null) {
             if (device != null) {
@@ -596,7 +602,7 @@ public class SegmentRoutingManager implements SegmentRoutingService {
                 processDeviceAdded(device);
                 processDeviceAdded(device);
                 groupHandler = groupHandlerMap.get(link.src()
                 groupHandler = groupHandlerMap.get(link.src()
                                                    .deviceId());
                                                    .deviceId());
-                groupHandler.linkUp(link);
+                groupHandler.linkUp(link, mastershipService.isLocalMaster(device.id()));
             }
             }
         }
         }
 
 

@@ -194,7 +194,7 @@ public class TunnelHandler {
             tunnel.allowToRemoveGroup(true);
             tunnel.allowToRemoveGroup(true);
         }
         }
 
 
-        return groupHandlerMap.get(deviceId).getNextObjectiveId(ns);
+        return groupHandlerMap.get(deviceId).getNextObjectiveId(ns, null);
     }
     }
 
 
 }
 }

@@ -93,7 +93,7 @@ public class DefaultEdgeGroupHandler extends DefaultGroupHandler {
                 + "with label for sw {} is {}",
                 + "with label for sw {} is {}",
                 deviceId, nsSet);
                 deviceId, nsSet);
 
 
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
     }
 
 
     @Override
     @Override
@@ -107,7 +107,7 @@ public class DefaultEdgeGroupHandler extends DefaultGroupHandler {
         Set<NeighborSet> nsSet = computeImpactedNeighborsetForPortEvent(
         Set<NeighborSet> nsSet = computeImpactedNeighborsetForPortEvent(
                                              newNeighborLink.dst().deviceId(),
                                              newNeighborLink.dst().deviceId(),
                                              devicePortMap.keySet());
                                              devicePortMap.keySet());
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
     }
 
 
     @Override
     @Override

@@ -21,11 +21,11 @@ import static org.slf4j.LoggerFactory.getLogger;
 import java.net.URI;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.List;
 import java.util.Map;
 import java.util.Map;
 import java.util.Set;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 import java.util.stream.Collectors;
 
 
 import org.onlab.packet.Ip4Prefix;
 import org.onlab.packet.Ip4Prefix;
@@ -38,6 +38,7 @@ import org.onosproject.net.DeviceId;
 import org.onosproject.net.Link;
 import org.onosproject.net.Link;
 import org.onosproject.net.PortNumber;
 import org.onosproject.net.PortNumber;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flowobjective.DefaultNextObjective;
 import org.onosproject.net.flowobjective.DefaultNextObjective;
 import org.onosproject.net.flowobjective.FlowObjectiveService;
 import org.onosproject.net.flowobjective.FlowObjectiveService;
@@ -71,10 +72,10 @@ public class DefaultGroupHandler {
     protected LinkService linkService;
     protected LinkService linkService;
     protected FlowObjectiveService flowObjectiveService;
     protected FlowObjectiveService flowObjectiveService;
 
 
-    protected HashMap<DeviceId, Set<PortNumber>> devicePortMap = new HashMap<>();
-    protected HashMap<PortNumber, DeviceId> portDeviceMap = new HashMap<>();
-    //protected HashMap<NeighborSet, Integer> deviceNextObjectiveIds =
-    //        new HashMap<NeighborSet, Integer>();
+    protected ConcurrentHashMap<DeviceId, Set<PortNumber>> devicePortMap =
+            new ConcurrentHashMap<>();
+    protected ConcurrentHashMap<PortNumber, DeviceId> portDeviceMap =
+            new ConcurrentHashMap<>();
     protected EventuallyConsistentMap<
     protected EventuallyConsistentMap<
         NeighborSetNextObjectiveStoreKey, Integer> nsNextObjStore = null;
         NeighborSetNextObjectiveStoreKey, Integer> nsNextObjStore = null;
     protected EventuallyConsistentMap<
     protected EventuallyConsistentMap<
@@ -173,7 +174,7 @@ public class DefaultGroupHandler {
      *
      *
      * @param newLink new neighbor link
      * @param newLink new neighbor link
      */
      */
-    public void linkUp(Link newLink) {
+    public void linkUp(Link newLink, boolean isMaster) {
 
 
         if (newLink.type() != Link.Type.DIRECT) {
         if (newLink.type() != Link.Type.DIRECT) {
             log.warn("linkUp: unknown link type");
             log.warn("linkUp: unknown link type");
@@ -186,6 +187,8 @@ public class DefaultGroupHandler {
             return;
             return;
         }
         }
 
 
+        log.info("* LinkUP: Device {} linkUp at local port {} to neighbor {}", deviceId,
+                 newLink.src().port(), newLink.dst().deviceId());
         MacAddress dstMac;
         MacAddress dstMac;
         try {
         try {
             dstMac = deviceConfig.getDeviceMac(newLink.dst().deviceId());
             dstMac = deviceConfig.getDeviceMac(newLink.dst().deviceId());
@@ -194,8 +197,6 @@ public class DefaultGroupHandler {
             return;
             return;
         }
         }
 
 
-        log.debug("Device {} linkUp at local port {} to neighbor {}", deviceId,
-                  newLink.src().port(), newLink.dst().deviceId());
         addNeighborAtPort(newLink.dst().deviceId(),
         addNeighborAtPort(newLink.dst().deviceId(),
                           newLink.src().port());
                           newLink.src().port());
         /*if (devicePortMap.get(newLink.dst().deviceId()) == null) {
         /*if (devicePortMap.get(newLink.dst().deviceId()) == null) {
@@ -237,15 +238,21 @@ public class DefaultGroupHandler {
 
 
                 nextObjBuilder.addTreatment(tBuilder.build());
                 nextObjBuilder.addTreatment(tBuilder.build());
 
 
-                log.debug("linkUp in device {}: Adding Bucket "
-                        + "with Port {} to next object id {}",
+                log.info("**linkUp in device {}: Adding Bucket "
+                        + "with Port {} to next object id {} and amIMaster:{}",
                         deviceId,
                         deviceId,
                         newLink.src().port(),
                         newLink.src().port(),
-                        nextId);
+                        nextId, isMaster);
+
+                if (isMaster) {
                     NextObjective nextObjective = nextObjBuilder.
                     NextObjective nextObjective = nextObjBuilder.
-                        add(new SRNextObjectiveContext(deviceId));
+                            addToExisting(new SRNextObjectiveContext(deviceId));
                     flowObjectiveService.next(deviceId, nextObjective);
                     flowObjectiveService.next(deviceId, nextObjective);
                 }
                 }
+            } else {
+                log.warn("linkUp in device {}, but global store has no record "
+                        + "for neighbor-set {}", deviceId, ns);
+            }
         }
         }
     }
     }
 
 
@@ -305,15 +312,16 @@ public class DefaultGroupHandler {
 
 
                 nextObjBuilder.addTreatment(tBuilder.build());
                 nextObjBuilder.addTreatment(tBuilder.build());
 
 
-                log.debug("portDown in device {}: Removing Bucket "
+                log.info("**portDown in device {}: Removing Bucket "
                         + "with Port {} to next object id {}",
                         + "with Port {} to next object id {}",
                         deviceId,
                         deviceId,
                         port,
                         port,
                         nextId);
                         nextId);
-                NextObjective nextObjective = nextObjBuilder.
+                // should do removefromexisting and only if master
+                /*NextObjective nextObjective = nextObjBuilder.
                         remove(new SRNextObjectiveContext(deviceId));
                         remove(new SRNextObjectiveContext(deviceId));
 
 
-                flowObjectiveService.next(deviceId, nextObjective);
+                flowObjectiveService.next(deviceId, nextObjective);*/
             }
             }
 
 
         }
         }
@@ -325,12 +333,15 @@ public class DefaultGroupHandler {
     /**
     /**
      * Returns the next objective associated with the neighborset.
      * Returns the next objective associated with the neighborset.
      * If there is no next objective for this neighborset, this API
      * If there is no next objective for this neighborset, this API
-     * would create a next objective and return.
+     * would create a next objective and return. Optionally metadata can be
+     * passed in for the creation of the next objective.
      *
      *
      * @param ns neighborset
      * @param ns neighborset
-     * @return int if found or -1
+     * @param meta metadata passed into the creation of a Next Objective
+     * @return int if found or -1 if there are errors in the creation of the
+     *          neighbor set.
      */
      */
-    public int getNextObjectiveId(NeighborSet ns) {
+    public int getNextObjectiveId(NeighborSet ns, TrafficSelector meta) {
         Integer nextId = nsNextObjStore.
         Integer nextId = nsNextObjStore.
                 get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
                 get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
         if (nextId == null) {
         if (nextId == null) {
@@ -343,7 +354,7 @@ public class DefaultGroupHandler {
                       .filter((nsStoreEntry) ->
                       .filter((nsStoreEntry) ->
                       (nsStoreEntry.getKey().deviceId().equals(deviceId)))
                       (nsStoreEntry.getKey().deviceId().equals(deviceId)))
                       .collect(Collectors.toList()));
                       .collect(Collectors.toList()));
-            createGroupsFromNeighborsets(Collections.singleton(ns));
+            createGroupsFromNeighborsets(Collections.singleton(ns), meta);
             nextId = nsNextObjStore.
             nextId = nsNextObjStore.
                     get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
                     get(new NeighborSetNextObjectiveStoreKey(deviceId, ns));
             if (nextId == null) {
             if (nextId == null) {
@@ -421,17 +432,19 @@ public class DefaultGroupHandler {
         // Update DeviceToPort database
         // Update DeviceToPort database
         log.debug("Device {} addNeighborAtPort: neighbor {} at port {}",
         log.debug("Device {} addNeighborAtPort: neighbor {} at port {}",
                   deviceId, neighborId, portToNeighbor);
                   deviceId, neighborId, portToNeighbor);
-        if (devicePortMap.get(neighborId) != null) {
-            devicePortMap.get(neighborId).add(portToNeighbor);
-        } else {
-            Set<PortNumber> ports = new HashSet<>();
+        Set<PortNumber> ports = Collections
+                .newSetFromMap(new ConcurrentHashMap<PortNumber, Boolean>());
         ports.add(portToNeighbor);
         ports.add(portToNeighbor);
-            devicePortMap.put(neighborId, ports);
+        Set<PortNumber> portnums = devicePortMap.putIfAbsent(neighborId, ports);
+        if (portnums != null) {
+            portnums.add(portToNeighbor);
         }
         }
 
 
         // Update portToDevice database
         // Update portToDevice database
-        if (portDeviceMap.get(portToNeighbor) == null) {
-            portDeviceMap.put(portToNeighbor, neighborId);
+        DeviceId prev = portDeviceMap.putIfAbsent(portToNeighbor, neighborId);
+        if (prev != null) {
+            log.warn("Device: {} port: {} has neighbor: {}. NOT updating "
+                    + "to neighbor: {}", deviceId, portToNeighbor, prev, neighborId);
         }
         }
     }
     }
 
 
@@ -505,61 +518,66 @@ public class DefaultGroupHandler {
      * Creates Groups from a set of NeighborSet given.
      * Creates Groups from a set of NeighborSet given.
      *
      *
      * @param nsSet a set of NeighborSet
      * @param nsSet a set of NeighborSet
+     * @param meta metadata passed into the creation of a Next Objective
      */
      */
-    public void createGroupsFromNeighborsets(Set<NeighborSet> nsSet) {
+    public void createGroupsFromNeighborsets(Set<NeighborSet> nsSet,
+                                             TrafficSelector meta) {
         for (NeighborSet ns : nsSet) {
         for (NeighborSet ns : nsSet) {
             int nextId = flowObjectiveService.allocateNextId();
             int nextId = flowObjectiveService.allocateNextId();
             NextObjective.Builder nextObjBuilder = DefaultNextObjective
             NextObjective.Builder nextObjBuilder = DefaultNextObjective
                     .builder().withId(nextId)
                     .builder().withId(nextId)
                     .withType(NextObjective.Type.HASHED).fromApp(appId);
                     .withType(NextObjective.Type.HASHED).fromApp(appId);
-            for (DeviceId d : ns.getDeviceIds()) {
-                if (devicePortMap.get(d) == null) {
-                    log.warn("Device {} is not in the port map yet", d);
+            for (DeviceId neighborId : ns.getDeviceIds()) {
+                if (devicePortMap.get(neighborId) == null) {
+                    log.warn("Neighbor {} is not in the port map yet for dev:{}",
+                             neighborId, deviceId);
                     return;
                     return;
-                } else if (devicePortMap.get(d).size() == 0) {
+                } else if (devicePortMap.get(neighborId).size() == 0) {
                     log.warn("There are no ports for "
                     log.warn("There are no ports for "
-                            + "the Device {} in the port map yet", d);
+                            + "the Device {} in the port map yet", neighborId);
                     return;
                     return;
                 }
                 }
 
 
-                MacAddress deviceMac;
+                MacAddress neighborMac;
                 try {
                 try {
-                    deviceMac = deviceConfig.getDeviceMac(d);
+                    neighborMac = deviceConfig.getDeviceMac(neighborId);
                 } catch (DeviceConfigNotFoundException e) {
                 } catch (DeviceConfigNotFoundException e) {
                     log.warn(e.getMessage() + " Aborting createGroupsFromNeighborsets.");
                     log.warn(e.getMessage() + " Aborting createGroupsFromNeighborsets.");
                     return;
                     return;
                 }
                 }
 
 
-                for (PortNumber sp : devicePortMap.get(d)) {
+                for (PortNumber sp : devicePortMap.get(neighborId)) {
                     TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment
                     TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment
                             .builder();
                             .builder();
-                    tBuilder.setOutput(sp)
-                            .setEthDst(deviceMac)
+                    tBuilder.setEthDst(neighborMac)
                             .setEthSrc(nodeMacAddr);
                             .setEthSrc(nodeMacAddr);
                     if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
                     if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
                         tBuilder.pushMpls()
                         tBuilder.pushMpls()
                                 .copyTtlOut()
                                 .copyTtlOut()
                                 .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
                                 .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
                     }
                     }
+                    tBuilder.setOutput(sp);
                     nextObjBuilder.addTreatment(tBuilder.build());
                     nextObjBuilder.addTreatment(tBuilder.build());
                 }
                 }
             }
             }
-
+            if (meta != null) {
+                nextObjBuilder.setMeta(meta);
+            }
             NextObjective nextObj = nextObjBuilder.
             NextObjective nextObj = nextObjBuilder.
                     add(new SRNextObjectiveContext(deviceId));
                     add(new SRNextObjectiveContext(deviceId));
-            flowObjectiveService.next(deviceId, nextObj);
-            log.debug("createGroupsFromNeighborsets: Submited "
+            log.info("**createGroupsFromNeighborsets: Submited "
                     + "next objective {} in device {}",
                     + "next objective {} in device {}",
                     nextId, deviceId);
                     nextId, deviceId);
+            flowObjectiveService.next(deviceId, nextObj);
             nsNextObjStore.put(new NeighborSetNextObjectiveStoreKey(deviceId, ns),
             nsNextObjStore.put(new NeighborSetNextObjectiveStoreKey(deviceId, ns),
                                nextId);
                                nextId);
         }
         }
     }
     }
 
 
+
     public void createGroupsFromSubnetConfig() {
     public void createGroupsFromSubnetConfig() {
         Map<Ip4Prefix, List<PortNumber>> subnetPortMap =
         Map<Ip4Prefix, List<PortNumber>> subnetPortMap =
                 this.deviceConfig.getSubnetPortsMap(this.deviceId);
                 this.deviceConfig.getSubnetPortsMap(this.deviceId);
-
         // Construct a broadcast group for each subnet
         // Construct a broadcast group for each subnet
         subnetPortMap.forEach((subnet, ports) -> {
         subnetPortMap.forEach((subnet, ports) -> {
             SubnetNextObjectiveStoreKey key =
             SubnetNextObjectiveStoreKey key =
@@ -612,6 +630,9 @@ public class DefaultGroupHandler {
                     .withType(NextObjective.Type.HASHED).fromApp(appId);
                     .withType(NextObjective.Type.HASHED).fromApp(appId);
             NextObjective nextObjective = nextObjBuilder.
             NextObjective nextObjective = nextObjBuilder.
                     remove(new SRNextObjectiveContext(deviceId));
                     remove(new SRNextObjectiveContext(deviceId));
+            log.info("**removeGroup: Submited "
+                    + "next objective {} in device {}",
+                    objectiveId, deviceId);
             flowObjectiveService.next(deviceId, nextObjective);
             flowObjectiveService.next(deviceId, nextObjective);
 
 
             for (Map.Entry<NeighborSetNextObjectiveStoreKey, Integer> entry: nsNextObjStore.entrySet()) {
             for (Map.Entry<NeighborSetNextObjectiveStoreKey, Integer> entry: nsNextObjStore.entrySet()) {
@@ -634,14 +655,14 @@ public class DefaultGroupHandler {
         }
         }
         @Override
         @Override
         public void onSuccess(Objective objective) {
         public void onSuccess(Objective objective) {
-            log.debug("Next objective operation successful in device {}",
-                      deviceId);
+            log.info("Next objective {} operation successful in device {}",
+                      objective.id(), deviceId);
         }
         }
 
 
         @Override
         @Override
         public void onError(Objective objective, ObjectiveError error) {
         public void onError(Objective objective, ObjectiveError error) {
             log.warn("Next objective {} operation failed with error: {} in device {}",
             log.warn("Next objective {} operation failed with error: {} in device {}",
-                     objective, error, deviceId);
+                     objective.id(), error, deviceId);
         }
         }
     }
     }
 }
 }

@@ -81,7 +81,7 @@ public class DefaultTransitGroupHandler extends DefaultGroupHandler {
         log.debug("createGroupsAtTransitRouter: The neighborset with label "
         log.debug("createGroupsAtTransitRouter: The neighborset with label "
                 + "for sw {} is {}", deviceId, nsSet);
                 + "for sw {} is {}", deviceId, nsSet);
 
 
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
     }
 
 
     @Override
     @Override
@@ -95,7 +95,7 @@ public class DefaultTransitGroupHandler extends DefaultGroupHandler {
         Set<NeighborSet> nsSet = computeImpactedNeighborsetForPortEvent(
         Set<NeighborSet> nsSet = computeImpactedNeighborsetForPortEvent(
                                              newNeighborLink.dst().deviceId(),
                                              newNeighborLink.dst().deviceId(),
                                              devicePortMap.keySet());
                                              devicePortMap.keySet());
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
     }
 
 
     @Override
     @Override

@@ -16,6 +16,7 @@
 package org.onosproject.net.flowobjective;
 package org.onosproject.net.flowobjective;
 
 
 import com.google.common.annotations.Beta;
 import com.google.common.annotations.Beta;
+
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
@@ -119,6 +120,53 @@ public final class DefaultForwardingObjective implements ForwardingObjective {
         return context;
         return context;
     }
     }
 
 
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#hashCode()
+     */
+    @Override
+    public int hashCode() {
+        return Objects.hash(selector, flag, permanent,
+                            timeout, appId, priority, nextId,
+                            treatment, op);
+    }
+
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#equals(java.lang.Object)
+     */
+    @Override
+    public boolean equals(final Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof DefaultForwardingObjective)) {
+            return false;
+        }
+        final DefaultForwardingObjective other = (DefaultForwardingObjective) obj;
+        boolean nextEq = false, treatmentEq = false;
+        if (this.selector.equals(other.selector) &&
+                this.flag == other.flag &&
+                this.permanent == other.permanent &&
+                this.timeout == other.timeout &&
+                this.appId.equals(other.appId) &&
+                this.priority == other.priority &&
+                this.op == other.op) {
+            if (this.nextId != null && other.nextId != null) {
+                nextEq = this.nextId == other.nextId;
+            }
+            if (this.treatment != null && other.treatment != null) {
+                treatmentEq = this.treatment.equals(other.treatment);
+            }
+            if (nextEq && treatmentEq) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
     /**
      * Returns a new builder.
      * Returns a new builder.
      *
      *

@@ -18,6 +18,7 @@ package org.onosproject.net.flowobjective;
 import com.google.common.annotations.Beta;
 import com.google.common.annotations.Beta;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableList;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.ApplicationId;
+import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
 
 
 import java.util.Collection;
 import java.util.Collection;
@@ -39,6 +40,7 @@ public final class DefaultNextObjective implements NextObjective {
     private final Integer id;
     private final Integer id;
     private final Operation op;
     private final Operation op;
     private final Optional<ObjectiveContext> context;
     private final Optional<ObjectiveContext> context;
+    private final TrafficSelector meta;
 
 
     private DefaultNextObjective(Builder builder) {
     private DefaultNextObjective(Builder builder) {
         this.treatments = builder.treatments;
         this.treatments = builder.treatments;
@@ -47,6 +49,7 @@ public final class DefaultNextObjective implements NextObjective {
         this.id = builder.id;
         this.id = builder.id;
         this.op = builder.op;
         this.op = builder.op;
         this.context = Optional.ofNullable(builder.context);
         this.context = Optional.ofNullable(builder.context);
+        this.meta = builder.meta;
     }
     }
 
 
     @Override
     @Override
@@ -94,6 +97,11 @@ public final class DefaultNextObjective implements NextObjective {
         return context;
         return context;
     }
     }
 
 
+    @Override
+    public TrafficSelector meta() {
+        return meta;
+    }
+
     /**
     /**
      * Returns a new builder.
      * Returns a new builder.
      *
      *
@@ -111,6 +119,7 @@ public final class DefaultNextObjective implements NextObjective {
         private List<TrafficTreatment> treatments;
         private List<TrafficTreatment> treatments;
         private Operation op;
         private Operation op;
         private ObjectiveContext context;
         private ObjectiveContext context;
+        private TrafficSelector meta;
 
 
         private final ImmutableList.Builder<TrafficTreatment> listBuilder
         private final ImmutableList.Builder<TrafficTreatment> listBuilder
                 = ImmutableList.builder();
                 = ImmutableList.builder();
@@ -172,6 +181,12 @@ public final class DefaultNextObjective implements NextObjective {
         }
         }
 
 
         @Override
         @Override
+        public Builder setMeta(TrafficSelector meta) {
+            this.meta = meta;
+            return this;
+        }
+
+        @Override
         public NextObjective add() {
         public NextObjective add() {
             treatments = listBuilder.build();
             treatments = listBuilder.build();
             op = Operation.ADD;
             op = Operation.ADD;
@@ -218,5 +233,55 @@ public final class DefaultNextObjective implements NextObjective {
 
 
             return new DefaultNextObjective(this);
             return new DefaultNextObjective(this);
         }
         }
+
+        @Override
+        public NextObjective addToExisting() {
+            treatments = listBuilder.build();
+            op = Operation.ADD_TO_EXISTING;
+            checkNotNull(appId, "Must supply an application id");
+            checkNotNull(id, "id cannot be null");
+            checkNotNull(type, "The type cannot be null");
+            checkArgument(!treatments.isEmpty(), "Must have at least one treatment");
+
+            return new DefaultNextObjective(this);
+        }
+
+        @Override
+        public NextObjective removeFromExisting() {
+            treatments = listBuilder.build();
+            op = Operation.REMOVE_FROM_EXISTING;
+            checkNotNull(appId, "Must supply an application id");
+            checkNotNull(id, "id cannot be null");
+            checkNotNull(type, "The type cannot be null");
+
+            return new DefaultNextObjective(this);
         }
         }
+
+        @Override
+        public NextObjective addToExisting(ObjectiveContext context) {
+            treatments = listBuilder.build();
+            op = Operation.ADD_TO_EXISTING;
+            this.context = context;
+            checkNotNull(appId, "Must supply an application id");
+            checkNotNull(id, "id cannot be null");
+            checkNotNull(type, "The type cannot be null");
+            checkArgument(!treatments.isEmpty(), "Must have at least one treatment");
+
+            return new DefaultNextObjective(this);
+        }
+
+        @Override
+        public NextObjective removeFromExisting(ObjectiveContext context) {
+            treatments = listBuilder.build();
+            op = Operation.REMOVE_FROM_EXISTING;
+            this.context = context;
+            checkNotNull(appId, "Must supply an application id");
+            checkNotNull(id, "id cannot be null");
+            checkNotNull(type, "The type cannot be null");
+
+            return new DefaultNextObjective(this);
+        }
+
+    }
+
 }
 }

@@ -17,6 +17,7 @@ package org.onosproject.net.flowobjective;
 
 
 import com.google.common.annotations.Beta;
 import com.google.common.annotations.Beta;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.ApplicationId;
+import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
 
 
 import java.util.Collection;
 import java.util.Collection;
@@ -34,7 +35,7 @@ import java.util.Collection;
  * - Failover
  * - Failover
  * - Simple
  * - Simple
  *
  *
- * These types will indicate to the driver what the intended behaviour is.
+ * These types will indicate to the driver what the intended behavior is.
  * For example, a broadcast next objective with a collection of output
  * For example, a broadcast next objective with a collection of output
  * treatments will indicate to a driver that all output actions are expected
  * treatments will indicate to a driver that all output actions are expected
  * to be executed simultaneously. The driver is then free to implement this
  * to be executed simultaneously. The driver is then free to implement this
@@ -84,6 +85,16 @@ public interface NextObjective extends Objective {
     Type type();
     Type type();
 
 
     /**
     /**
+     * Auxiliary optional information provided to the device-driver.Typically
+     * conveys information about selectors (matches) that are intended to
+     * use this Next Objective.
+     *
+     * @return a selector intended to pass meta information to the device driver.
+     *         Value may be null if no meta information is provided.
+     */
+    TrafficSelector meta();
+
+    /**
      * A next step builder.
      * A next step builder.
      */
      */
     interface Builder extends Objective.Builder {
     interface Builder extends Objective.Builder {
@@ -131,6 +142,14 @@ public interface NextObjective extends Objective {
         Builder withPriority(int priority);
         Builder withPriority(int priority);
 
 
         /**
         /**
+         * Set meta information related to this next objective.
+         *
+         * @param selector match conditions
+         * @return an objective builder
+         */
+        Builder setMeta(TrafficSelector selector);
+
+        /**
          * Builds the next objective that will be added.
          * Builds the next objective that will be added.
          *
          *
          * @return a next objective
          * @return a next objective
@@ -162,6 +181,40 @@ public interface NextObjective extends Objective {
          */
          */
         NextObjective remove(ObjectiveContext context);
         NextObjective remove(ObjectiveContext context);
 
 
+        /**
+         * Build the next objective that will be added, with {@link Operation}
+         * ADD_TO_EXISTING.
+         *
+         * @return a next objective
+         */
+        NextObjective addToExisting();
+
+        /**
+         * Build the next objective that will be removed, with {@link Operation}
+         * REMOVE_FROM_EXISTING.
+         *
+         * @return a next objective
+         */
+        NextObjective removeFromExisting();
+
+        /**
+         * Builds the next objective that will be added, with {@link Operation}
+         * ADD_TO_EXISTING. The context will be used to notify the calling application.
+         *
+         * @param context an objective context
+         * @return a next objective
+         */
+        NextObjective addToExisting(ObjectiveContext context);
+
+        /**
+         * Builds the next objective that will be removed, with {@link Operation}
+         * REMOVE_FROM_EXISTING. The context will be used to notify the calling application.
+         *
+         * @param context an objective context
+         * @return a next objective
+         */
+        NextObjective removeFromExisting(ObjectiveContext context);
+
     }
     }
 
 
 }
 }

@@ -21,7 +21,7 @@ import org.onosproject.core.ApplicationId;
 import java.util.Optional;
 import java.util.Optional;
 
 
 /**
 /**
- * Base representation of an flow description.
+ * Base representation of a flow-objective description.
  */
  */
 @Beta
 @Beta
 public interface Objective {
 public interface Objective {
@@ -35,14 +35,30 @@ public interface Objective {
      */
      */
     enum Operation {
     enum Operation {
         /**
         /**
-         * Adds the objective.
+         * Adds the objective. Can be used for any flow objective. For forwarding
+         * and filtering objectives, existing objectives with identical selector
+         * and priority fields (but different treatments or next) will be replaced.
+         * For next objectives, if modification is desired, ADD will not
+         * do anything - use ADD_TO_EXISTING.
          */
          */
         ADD,
         ADD,
 
 
         /**
         /**
-         * Removes the objective.
+         * Removes the objective. Can be used for any flow objective.
          */
          */
-        REMOVE
+        REMOVE,
+
+        /**
+         * Add to an existing Next Objective. Should not be used for any other
+         * objective.
+         */
+        ADD_TO_EXISTING,
+
+        /**
+         * Remove from an existing Next Objective. Should not be used for any
+         * other objective.
+         */
+        REMOVE_FROM_EXISTING
     }
     }
 
 
     /**
     /**
@@ -129,6 +145,7 @@ public interface Objective {
          * @return an objective builder
          * @return an objective builder
          */
          */
         Builder withPriority(int priority);
         Builder withPriority(int priority);
+
     }
     }
 
 
 }
 }

@@ -25,6 +25,7 @@ import java.util.Arrays;
 public class DefaultGroupKey implements GroupKey {
 public class DefaultGroupKey implements GroupKey {
 
 
     private final byte[] key;
     private final byte[] key;
+    protected static final char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray();
 
 
     public DefaultGroupKey(byte[] key) {
     public DefaultGroupKey(byte[] key) {
         this.key = checkNotNull(key);
         this.key = checkNotNull(key);
@@ -52,4 +53,20 @@ public class DefaultGroupKey implements GroupKey {
         return Arrays.hashCode(this.key);
         return Arrays.hashCode(this.key);
     }
     }
 
 
+    /**
+     * Returns a hex string representation of the byte array that is used
+     * as a group key. This solution was adapted from
+     * http://stackoverflow.com/questions/9655181/
+     */
+    @Override
+    public String toString() {
+        char[] hexChars = new char[key.length * 2];
+        for (int j = 0; j < key.length; j++) {
+            int v = key[j] & 0xFF;
+            hexChars[j * 2] = HEX_ARRAY[v >>> 4];
+            hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F];
+        }
+        return "GroupKey:0x" + new String(hexChars);
+    }
+
 }
 }
\ No newline at end of file

@@ -54,6 +54,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;
 
 
 import java.util.Map;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ExecutorService;
 
 
@@ -226,10 +227,11 @@ public class FlowObjectiveManager implements FlowObjectiveService {
         if (fwd.nextId() != null &&
         if (fwd.nextId() != null &&
                 flowObjectiveStore.getNextGroup(fwd.nextId()) == null) {
                 flowObjectiveStore.getNextGroup(fwd.nextId()) == null) {
             log.trace("Queuing forwarding objective for nextId {}", fwd.nextId());
             log.trace("Queuing forwarding objective for nextId {}", fwd.nextId());
-            if (pendingForwards.putIfAbsent(fwd.nextId(),
-                                            Sets.newHashSet(new PendingNext(deviceId, fwd))) != null) {
-                Set<PendingNext> pending = pendingForwards.get(fwd.nextId());
-                pending.add(new PendingNext(deviceId, fwd));
+            // TODO: change to computeIfAbsent
+            Set<PendingNext> pnext = pendingForwards.putIfAbsent(fwd.nextId(),
+                                         Sets.newHashSet(new PendingNext(deviceId, fwd)));
+            if (pnext != null) {
+                pnext.add(new PendingNext(deviceId, fwd));
             }
             }
             return true;
             return true;
         }
         }
@@ -412,5 +414,26 @@ public class FlowObjectiveManager implements FlowObjectiveService {
         public ForwardingObjective forwardingObjective() {
         public ForwardingObjective forwardingObjective() {
             return fwd;
             return fwd;
         }
         }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(deviceId, fwd);
+        }
+
+        @Override
+        public boolean equals(final Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (!(obj instanceof PendingNext)) {
+                return false;
+            }
+            final PendingNext other = (PendingNext) obj;
+            if (this.deviceId.equals(other.deviceId) &&
+                    this.fwd.equals(other.fwd)) {
+                return true;
+            }
+            return false;
+        }
     }
     }
 }
 }

@@ -348,9 +348,11 @@ public class DistributedGroupStore
     public void storeGroupDescription(GroupDescription groupDesc) {
     public void storeGroupDescription(GroupDescription groupDesc) {
         log.debug("In storeGroupDescription");
         log.debug("In storeGroupDescription");
         // Check if a group is existing with the same key
         // Check if a group is existing with the same key
-        if (getGroup(groupDesc.deviceId(), groupDesc.appCookie()) != null) {
-            log.warn("Group already exists with the same key {}",
-                     groupDesc.appCookie());
+        Group existingGroup = getGroup(groupDesc.deviceId(), groupDesc.appCookie());
+        if (existingGroup != null) {
+            log.warn("Group already exists with the same key {} in dev:{} with id:{}",
+                     groupDesc.appCookie(), groupDesc.deviceId(),
+                     Integer.toHexString(existingGroup.id().id()));
             return;
             return;
         }
         }
 
 

@@ -368,6 +368,7 @@ public final class KryoNamespaces {
                     L2ModificationInstruction.ModVlanPcpInstruction.class,
                     L2ModificationInstruction.ModVlanPcpInstruction.class,
                     L2ModificationInstruction.PopVlanInstruction.class,
                     L2ModificationInstruction.PopVlanInstruction.class,
                     L2ModificationInstruction.ModMplsLabelInstruction.class,
                     L2ModificationInstruction.ModMplsLabelInstruction.class,
+                    L2ModificationInstruction.ModMplsBosInstruction.class,
                     L2ModificationInstruction.ModMplsTtlInstruction.class,
                     L2ModificationInstruction.ModMplsTtlInstruction.class,
                     L2ModificationInstruction.ModTunnelIdInstruction.class,
                     L2ModificationInstruction.ModTunnelIdInstruction.class,
                     L3ModificationInstruction.class,
                     L3ModificationInstruction.class,

@@ -18,15 +18,19 @@ package org.onosproject.driver.pipeline;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.slf4j.LoggerFactory.getLogger;
 
 
 import java.util.ArrayList;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Collections;
+import java.util.Deque;
 import java.util.List;
 import java.util.List;
 import java.util.Set;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentHashMap;
 
 
+import org.onlab.packet.Ethernet;
 import org.onlab.packet.VlanId;
 import org.onlab.packet.VlanId;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.net.Port;
 import org.onosproject.net.Port;
 import org.onosproject.net.PortNumber;
 import org.onosproject.net.PortNumber;
+import org.onosproject.net.behaviour.NextGroup;
 import org.onosproject.net.flow.DefaultFlowRule;
 import org.onosproject.net.flow.DefaultFlowRule;
 import org.onosproject.net.flow.DefaultTrafficSelector;
 import org.onosproject.net.flow.DefaultTrafficSelector;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
@@ -35,8 +39,18 @@ import org.onosproject.net.flow.FlowRuleOperations;
 import org.onosproject.net.flow.FlowRuleOperationsContext;
 import org.onosproject.net.flow.FlowRuleOperationsContext;
 import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.criteria.Criterion;
+import org.onosproject.net.flow.criteria.EthTypeCriterion;
+import org.onosproject.net.flow.criteria.IPCriterion;
+import org.onosproject.net.flow.criteria.MplsBosCriterion;
+import org.onosproject.net.flow.criteria.MplsCriterion;
 import org.onosproject.net.flow.criteria.PortCriterion;
 import org.onosproject.net.flow.criteria.PortCriterion;
 import org.onosproject.net.flow.criteria.VlanIdCriterion;
 import org.onosproject.net.flow.criteria.VlanIdCriterion;
+import org.onosproject.net.flow.instructions.Instruction;
+import org.onosproject.net.flowobjective.ForwardingObjective;
+import org.onosproject.net.flowobjective.ObjectiveError;
+import org.onosproject.net.group.Group;
+import org.onosproject.net.group.GroupKey;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
 
 
@@ -108,6 +122,81 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
         return rules;
         return rules;
     }
     }
 
 
+    @Override
+    protected Collection<FlowRule> processSpecific(ForwardingObjective fwd) {
+        TrafficSelector selector = fwd.selector();
+        EthTypeCriterion ethType =
+                (EthTypeCriterion) selector.getCriterion(Criterion.Type.ETH_TYPE);
+        if ((ethType == null) ||
+                (ethType.ethType().toShort() != Ethernet.TYPE_IPV4) &&
+                (ethType.ethType().toShort() != Ethernet.MPLS_UNICAST)) {
+            log.warn("processSpecific: Unsupported "
+                    + "forwarding objective criteraia");
+            fail(fwd, ObjectiveError.UNSUPPORTED);
+            return Collections.emptySet();
+        }
+
+        int forTableId = -1;
+        TrafficSelector.Builder filteredSelector = DefaultTrafficSelector.builder();
+        if (ethType.ethType().toShort() == Ethernet.TYPE_IPV4) {
+            filteredSelector.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(((IPCriterion)
+                        selector.getCriterion(Criterion.Type.IPV4_DST)).ip());
+            forTableId = UNICAST_ROUTING_TABLE;
+            log.debug("processing IPv4 specific forwarding objective {} hash{} in dev:{}",
+                      fwd.id(), fwd.hashCode(), deviceId);
+        } else {
+            filteredSelector
+                .matchEthType(Ethernet.MPLS_UNICAST)
+                .matchMplsLabel(((MplsCriterion)
+                        selector.getCriterion(Criterion.Type.MPLS_LABEL)).label());
+            MplsBosCriterion bos = (MplsBosCriterion) selector
+                                        .getCriterion(Criterion.Type.MPLS_BOS);
+            if (bos != null) {
+                filteredSelector.matchMplsBos(bos.mplsBos());
+            }
+            forTableId = MPLS_TABLE_1;
+            log.debug("processing MPLS specific forwarding objective {} hash:{} in dev {}",
+                      fwd.id(), fwd.hashCode(), deviceId);
+        }
+
+        TrafficTreatment.Builder tb = DefaultTrafficTreatment.builder();
+        if (fwd.treatment() != null) {
+            for (Instruction i : fwd.treatment().allInstructions()) {
+                tb.add(i);
+            }
+        }
+
+        if (fwd.nextId() != null) {
+            NextGroup next = flowObjectiveStore.getNextGroup(fwd.nextId());
+            List<Deque<GroupKey>> gkeys = appKryo.deserialize(next.data());
+            // we only need the top level group's key to point the flow to it
+            Group group = groupService.getGroup(deviceId, gkeys.get(0).peekFirst());
+            if (group == null) {
+                log.warn("The group left!");
+                fail(fwd, ObjectiveError.GROUPMISSING);
+                return Collections.emptySet();
+            }
+            tb.deferred().group(group.id());
+        }
+        tb.transition(ACL_TABLE);
+        FlowRule.Builder ruleBuilder = DefaultFlowRule.builder()
+                .fromApp(fwd.appId())
+                .withPriority(fwd.priority())
+                .forDevice(deviceId)
+                .withSelector(filteredSelector.build())
+                .withTreatment(tb.build())
+                .forTable(forTableId);
+
+        if (fwd.permanent()) {
+            ruleBuilder.makePermanent();
+        } else {
+            ruleBuilder.makeTemporary(fwd.timeout());
+        }
+
+        return Collections.singletonList(ruleBuilder.build());
+    }
+
 
 
     @Override
     @Override
     protected void initializePipeline() {
     protected void initializePipeline() {

@@ -19,9 +19,11 @@ import static org.onlab.util.Tools.groupedThreads;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.slf4j.LoggerFactory.getLogger;
 
 
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Collections;
+import java.util.Deque;
 import java.util.List;
 import java.util.List;
 import java.util.Map;
 import java.util.Map;
 import java.util.Set;
 import java.util.Set;
@@ -65,15 +67,20 @@ import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.criteria.Criteria;
 import org.onosproject.net.flow.criteria.Criteria;
 import org.onosproject.net.flow.criteria.Criterion;
 import org.onosproject.net.flow.criteria.Criterion;
+import org.onosproject.net.flow.criteria.Criterion.Type;
 import org.onosproject.net.flow.criteria.EthCriterion;
 import org.onosproject.net.flow.criteria.EthCriterion;
 import org.onosproject.net.flow.criteria.EthTypeCriterion;
 import org.onosproject.net.flow.criteria.EthTypeCriterion;
 import org.onosproject.net.flow.criteria.IPCriterion;
 import org.onosproject.net.flow.criteria.IPCriterion;
+import org.onosproject.net.flow.criteria.MplsBosCriterion;
+import org.onosproject.net.flow.criteria.MplsCriterion;
 import org.onosproject.net.flow.criteria.PortCriterion;
 import org.onosproject.net.flow.criteria.PortCriterion;
 import org.onosproject.net.flow.criteria.VlanIdCriterion;
 import org.onosproject.net.flow.criteria.VlanIdCriterion;
 import org.onosproject.net.flow.instructions.Instruction;
 import org.onosproject.net.flow.instructions.Instruction;
 import org.onosproject.net.flow.instructions.Instructions.OutputInstruction;
 import org.onosproject.net.flow.instructions.Instructions.OutputInstruction;
 import org.onosproject.net.flow.instructions.L2ModificationInstruction;
 import org.onosproject.net.flow.instructions.L2ModificationInstruction;
+import org.onosproject.net.flow.instructions.L2ModificationInstruction.L2SubType;
 import org.onosproject.net.flow.instructions.L2ModificationInstruction.ModEtherInstruction;
 import org.onosproject.net.flow.instructions.L2ModificationInstruction.ModEtherInstruction;
+import org.onosproject.net.flow.instructions.L2ModificationInstruction.ModMplsLabelInstruction;
 import org.onosproject.net.flow.instructions.L2ModificationInstruction.ModVlanIdInstruction;
 import org.onosproject.net.flow.instructions.L2ModificationInstruction.ModVlanIdInstruction;
 import org.onosproject.net.flowobjective.FilteringObjective;
 import org.onosproject.net.flowobjective.FilteringObjective;
 import org.onosproject.net.flowobjective.FlowObjectiveStore;
 import org.onosproject.net.flowobjective.FlowObjectiveStore;
@@ -128,50 +135,43 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
     protected static final int LOWEST_PRIORITY = 0x0;
     protected static final int LOWEST_PRIORITY = 0x0;
 
 
     /*
     /*
-     * Group keys are normally generated by using the next Objective id. In the
-     * case of a next objective resulting in a group chain, each group derives a
-     * group key from the next objective id in the following way:
-     * The upper 4 bits of the group-key are used to denote the position of the
-     * group in the group chain. For example, in the chain
-     * group0 --> group1 --> group2 --> port
-     * group0's group key would have the upper 4 bits as 0, group1's upper four
-     * bits would be 1, and so on
-     */
-    private static final int GROUP0MASK = 0x0;
-    private static final int GROUP1MASK = 0x10000000;
-
-    /*
      * OFDPA requires group-id's to have a certain form.
      * OFDPA requires group-id's to have a certain form.
      * L2 Interface Groups have <4bits-0><12bits-vlanid><16bits-portid>
      * L2 Interface Groups have <4bits-0><12bits-vlanid><16bits-portid>
      * L3 Unicast Groups have <4bits-2><28bits-index>
      * L3 Unicast Groups have <4bits-2><28bits-index>
+     * MPLS Interface Groups have <4bits-9><4bits:0><24bits-index>
+     * L3 ECMP Groups have <4bits-7><28bits-index>
+     * L2 Flood Groups have <4bits-4><12bits-vlanid><16bits-index>
+     * L3 VPN Groups have <4bits-9><4bits-2><24bits-index>
      */
      */
     private static final int L2INTERFACEMASK = 0x0;
     private static final int L2INTERFACEMASK = 0x0;
     private static final int L3UNICASTMASK = 0x20000000;
     private static final int L3UNICASTMASK = 0x20000000;
-    //private static final int MPLSINTERFACEMASK = 0x90000000;
+    private static final int MPLSINTERFACEMASK = 0x90000000;
     private static final int L3ECMPMASK = 0x70000000;
     private static final int L3ECMPMASK = 0x70000000;
     private static final int L2FLOODMASK = 0x40000000;
     private static final int L2FLOODMASK = 0x40000000;
+    private static final int L3VPNMASK = 0x92000000;
 
 
     private final Logger log = getLogger(getClass());
     private final Logger log = getLogger(getClass());
     private ServiceDirectory serviceDirectory;
     private ServiceDirectory serviceDirectory;
     protected FlowRuleService flowRuleService;
     protected FlowRuleService flowRuleService;
     private CoreService coreService;
     private CoreService coreService;
-    private GroupService groupService;
-    private FlowObjectiveStore flowObjectiveStore;
+    protected GroupService groupService;
+    protected FlowObjectiveStore flowObjectiveStore;
     protected DeviceId deviceId;
     protected DeviceId deviceId;
     protected ApplicationId driverId;
     protected ApplicationId driverId;
     protected PacketService packetService;
     protected PacketService packetService;
     protected DeviceService deviceService;
     protected DeviceService deviceService;
     private InternalPacketProcessor processor = new InternalPacketProcessor();
     private InternalPacketProcessor processor = new InternalPacketProcessor();
-    private KryoNamespace appKryo = new KryoNamespace.Builder()
+    protected KryoNamespace appKryo = new KryoNamespace.Builder()
         .register(KryoNamespaces.API)
         .register(KryoNamespaces.API)
         .register(GroupKey.class)
         .register(GroupKey.class)
         .register(DefaultGroupKey.class)
         .register(DefaultGroupKey.class)
-        .register(OfdpaGroupChain.class)
+        .register(OfdpaNextGroup.class)
         .register(byte[].class)
         .register(byte[].class)
+        .register(ArrayDeque.class)
         .build();
         .build();
 
 
-    private Cache<GroupKey, OfdpaGroupChain> pendingNextObjectives;
-    private ConcurrentHashMap<GroupKey, GroupChainElem> pendingGroups;
+    private Cache<GroupKey, OfdpaNextGroup> pendingNextObjectives;
+    private ConcurrentHashMap<GroupKey, Set<GroupChainElem>> pendingGroups;
 
 
     private ScheduledExecutorService groupChecker =
     private ScheduledExecutorService groupChecker =
             Executors.newScheduledThreadPool(2, groupedThreads("onos/pipeliner",
             Executors.newScheduledThreadPool(2, groupedThreads("onos/pipeliner",
@@ -184,6 +184,8 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
     Map<VlanId, Set<PortNumber>> vlan2Port = new ConcurrentHashMap<VlanId,
     Map<VlanId, Set<PortNumber>> vlan2Port = new ConcurrentHashMap<VlanId,
                                                         Set<PortNumber>>();
                                                         Set<PortNumber>>();
 
 
+    // index number for group creation
+    AtomicInteger l3vpnindex = new AtomicInteger(0);
 
 
 
 
     @Override
     @Override
@@ -193,7 +195,8 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
 
 
         pendingNextObjectives = CacheBuilder.newBuilder()
         pendingNextObjectives = CacheBuilder.newBuilder()
                 .expireAfterWrite(20, TimeUnit.SECONDS)
                 .expireAfterWrite(20, TimeUnit.SECONDS)
-                .removalListener((RemovalNotification<GroupKey, OfdpaGroupChain> notification) -> {
+                .removalListener((
+                     RemovalNotification<GroupKey, OfdpaNextGroup> notification) -> {
                          if (notification.getCause() == RemovalCause.EXPIRED) {
                          if (notification.getCause() == RemovalCause.EXPIRED) {
                              fail(notification.getValue().nextObjective(),
                              fail(notification.getValue().nextObjective(),
                                   ObjectiveError.GROUPINSTALLATIONFAILED);
                                   ObjectiveError.GROUPINSTALLATIONFAILED);
@@ -201,7 +204,7 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
                 }).build();
                 }).build();
 
 
         groupChecker.scheduleAtFixedRate(new GroupChecker(), 0, 500, TimeUnit.MILLISECONDS);
         groupChecker.scheduleAtFixedRate(new GroupChecker(), 0, 500, TimeUnit.MILLISECONDS);
-        pendingGroups = new ConcurrentHashMap<GroupKey, GroupChainElem>();
+        pendingGroups = new ConcurrentHashMap<GroupKey, Set<GroupChainElem>>();
 
 
         coreService = serviceDirectory.get(CoreService.class);
         coreService = serviceDirectory.get(CoreService.class);
         flowRuleService = serviceDirectory.get(FlowRuleService.class);
         flowRuleService = serviceDirectory.get(FlowRuleService.class);
@@ -285,22 +288,49 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
 
 
     @Override
     @Override
     public void next(NextObjective nextObjective) {
     public void next(NextObjective nextObjective) {
-        log.debug("Processing NextObjective id{} op{}", nextObjective.id(),
-                  nextObjective.op());
-        if (nextObjective.op() == Objective.Operation.REMOVE) {
-            if (nextObjective.next().isEmpty()) {
-                removeGroup(nextObjective);
-            } else {
-                removeBucketFromGroup(nextObjective);
-            }
-        } else if (nextObjective.op() == Objective.Operation.ADD) {
         NextGroup nextGroup = flowObjectiveStore.getNextGroup(nextObjective.id());
         NextGroup nextGroup = flowObjectiveStore.getNextGroup(nextObjective.id());
+        switch (nextObjective.op()) {
+        case ADD:
+            if (nextGroup != null) {
+                log.warn("Cannot add next {} that already exists in device {}",
+                         nextObjective.id(), deviceId);
+                return;
+            }
+            log.debug("Processing NextObjective id{} in dev{} - add group",
+                      nextObjective.id(), deviceId);
+            addGroup(nextObjective);
+            break;
+        case ADD_TO_EXISTING:
             if (nextGroup != null) {
             if (nextGroup != null) {
+                log.debug("Processing NextObjective id{} in dev{} - add bucket",
+                          nextObjective.id(), deviceId);
                 addBucketToGroup(nextObjective);
                 addBucketToGroup(nextObjective);
             } else {
             } else {
-                addGroup(nextObjective);
+                // it is possible that group-chain has not been fully created yet
+                waitToAddBucketToGroup(nextObjective);
             }
             }
-        } else {
+            break;
+        case REMOVE:
+            if (nextGroup == null) {
+                log.warn("Cannot remove next {} that does not exist in device {}",
+                         nextObjective.id(), deviceId);
+                return;
+            }
+            log.debug("Processing NextObjective id{}  in dev{} - remove group",
+                      nextObjective.id(), deviceId);
+            removeGroup(nextObjective);
+            break;
+        case REMOVE_FROM_EXISTING:
+            if (nextGroup == null) {
+                log.warn("Cannot remove from next {} that does not exist in device {}",
+                         nextObjective.id(), deviceId);
+                return;
+            }
+            log.debug("Processing NextObjective id{} in dev{} - remove bucket",
+                      nextObjective.id(), deviceId);
+            removeBucketFromGroup(nextObjective);
+            break;
+        default:
             log.warn("Unsupported operation {}", nextObjective.op());
             log.warn("Unsupported operation {}", nextObjective.op());
         }
         }
     }
     }
@@ -309,7 +339,6 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
     //  Flow handling
     //  Flow handling
     //////////////////////////////////////
     //////////////////////////////////////
 
 
-
     /**
     /**
      * As per OFDPA 2.0 TTP, filtering of VLAN ids, MAC addresses (for routing)
      * As per OFDPA 2.0 TTP, filtering of VLAN ids, MAC addresses (for routing)
      * and IP addresses configured on switch ports happen in different tables.
      * and IP addresses configured on switch ports happen in different tables.
@@ -520,7 +549,6 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
     /**
     /**
      * Allows routed packets with correct destination MAC to be directed
      * Allows routed packets with correct destination MAC to be directed
      * to unicast-IP routing table or MPLS forwarding table.
      * to unicast-IP routing table or MPLS forwarding table.
-     * XXX need to add rule for multicast routing.
      *
      *
      * @param portCriterion  port on device for which this filter is programmed
      * @param portCriterion  port on device for which this filter is programmed
      * @param ethCriterion   dstMac of device for which is filter is programmed
      * @param ethCriterion   dstMac of device for which is filter is programmed
@@ -661,38 +689,78 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
 
 
     /**
     /**
      * In the OF-DPA 2.0 pipeline, specific forwarding refers to the IP table
      * In the OF-DPA 2.0 pipeline, specific forwarding refers to the IP table
-     * (unicast or multicast) or the L2 table (mac + vlan).
+     * (unicast or multicast) or the L2 table (mac + vlan) or the MPLS table.
      *
      *
      * @param fwd the forwarding objective of type 'specific'
      * @param fwd the forwarding objective of type 'specific'
      * @return    a collection of flow rules. Typically there will be only one
      * @return    a collection of flow rules. Typically there will be only one
      *            for this type of forwarding objective. An empty set may be
      *            for this type of forwarding objective. An empty set may be
      *            returned if there is an issue in processing the objective.
      *            returned if there is an issue in processing the objective.
      */
      */
-    private Collection<FlowRule> processSpecific(ForwardingObjective fwd) {
-        log.debug("Processing specific forwarding objective");
+    protected Collection<FlowRule> processSpecific(ForwardingObjective fwd) {
         TrafficSelector selector = fwd.selector();
         TrafficSelector selector = fwd.selector();
         EthTypeCriterion ethType =
         EthTypeCriterion ethType =
                 (EthTypeCriterion) selector.getCriterion(Criterion.Type.ETH_TYPE);
                 (EthTypeCriterion) selector.getCriterion(Criterion.Type.ETH_TYPE);
-        // XXX currently supporting only the L3 unicast table
-        if (ethType == null || ethType.ethType().toShort() != Ethernet.TYPE_IPV4) {
+        if ((ethType == null) ||
+                (ethType.ethType().toShort() != Ethernet.TYPE_IPV4) &&
+                (ethType.ethType().toShort() != Ethernet.MPLS_UNICAST)) {
+            log.warn("processSpecific: Unsupported "
+                    + "forwarding objective criteraia");
             fail(fwd, ObjectiveError.UNSUPPORTED);
             fail(fwd, ObjectiveError.UNSUPPORTED);
             return Collections.emptySet();
             return Collections.emptySet();
         }
         }
 
 
-        TrafficSelector filteredSelector =
-                DefaultTrafficSelector.builder()
-                        .matchEthType(Ethernet.TYPE_IPV4)
-                        .matchIPDst(
-                                ((IPCriterion)
-                                        selector.getCriterion(Criterion.Type.IPV4_DST)).ip())
-                        .build();
+        int forTableId = -1;
+        TrafficSelector.Builder filteredSelector = DefaultTrafficSelector.builder();
+        if (ethType.ethType().toShort() == Ethernet.TYPE_IPV4) {
+            filteredSelector.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(((IPCriterion)
+                        selector.getCriterion(Criterion.Type.IPV4_DST)).ip());
+            forTableId = UNICAST_ROUTING_TABLE;
+            log.debug("processing IPv4 specific forwarding objective {} in dev:{}",
+                      fwd.id(), deviceId);
+        } else {
+            filteredSelector
+                .matchEthType(Ethernet.MPLS_UNICAST)
+                .matchMplsLabel(((MplsCriterion)
+                        selector.getCriterion(Criterion.Type.MPLS_LABEL)).label());
+            MplsBosCriterion bos = (MplsBosCriterion) selector
+                                        .getCriterion(Criterion.Type.MPLS_BOS);
+            if (bos != null) {
+                filteredSelector.matchMplsBos(bos.mplsBos());
+            }
+            forTableId = MPLS_TABLE_1;
+            log.debug("processing MPLS specific forwarding objective {} in dev {}",
+                      fwd.id(), deviceId);
+        }
 
 
         TrafficTreatment.Builder tb = DefaultTrafficTreatment.builder();
         TrafficTreatment.Builder tb = DefaultTrafficTreatment.builder();
+        boolean popMpls = false;
+        if (fwd.treatment() != null) {
+            for (Instruction i : fwd.treatment().allInstructions()) {
+                tb.add(i);
+                if (i instanceof L2ModificationInstruction &&
+                    ((L2ModificationInstruction) i).subtype() == L2SubType.MPLS_POP) {
+                        popMpls = true;
+                }
+            }
+        }
 
 
         if (fwd.nextId() != null) {
         if (fwd.nextId() != null) {
+            if (forTableId == MPLS_TABLE_1 && !popMpls) {
+                log.warn("SR CONTINUE case cannot be handled as MPLS ECMP "
+                        + "is not implemented in OF-DPA yet. Aborting this flow "
+                        + "in this device {}", deviceId);
+                // XXX We could convert to forwarding to a single-port, via a
+                // MPLS interface, or a MPLS SWAP (with-same) but that would
+                // have to be handled in the next-objective. Also the pop-mpls
+                // logic used here won't work in non-BoS case.
+                return Collections.emptySet();
+            }
+
             NextGroup next = flowObjectiveStore.getNextGroup(fwd.nextId());
             NextGroup next = flowObjectiveStore.getNextGroup(fwd.nextId());
-            List<GroupKey> gkeys = appKryo.deserialize(next.data());
-            Group group = groupService.getGroup(deviceId, gkeys.get(0));
+            List<Deque<GroupKey>> gkeys = appKryo.deserialize(next.data());
+            // we only need the top level group's key to point the flow to it
+            Group group = groupService.getGroup(deviceId, gkeys.get(0).peekFirst());
             if (group == null) {
             if (group == null) {
                 log.warn("The group left!");
                 log.warn("The group left!");
                 fail(fwd, ObjectiveError.GROUPMISSING);
                 fail(fwd, ObjectiveError.GROUPMISSING);
@@ -705,8 +773,9 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
                 .fromApp(fwd.appId())
                 .fromApp(fwd.appId())
                 .withPriority(fwd.priority())
                 .withPriority(fwd.priority())
                 .forDevice(deviceId)
                 .forDevice(deviceId)
-                .withSelector(filteredSelector)
-                .withTreatment(tb.build());
+                .withSelector(filteredSelector.build())
+                .withTreatment(tb.build())
+                .forTable(forTableId);
 
 
         if (fwd.permanent()) {
         if (fwd.permanent()) {
             ruleBuilder.makePermanent();
             ruleBuilder.makePermanent();
@@ -714,7 +783,6 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
             ruleBuilder.makeTemporary(fwd.timeout());
             ruleBuilder.makeTemporary(fwd.timeout());
         }
         }
 
 
-        ruleBuilder.forTable(UNICAST_ROUTING_TABLE);
         return Collections.singletonList(ruleBuilder.build());
         return Collections.singletonList(ruleBuilder.build());
     }
     }
 
 
@@ -724,7 +792,7 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
         }
         }
     }
     }
 
 
-    private void fail(Objective obj, ObjectiveError error) {
+    protected void fail(Objective obj, ObjectiveError error) {
         if (obj.context().isPresent()) {
         if (obj.context().isPresent()) {
             obj.context().get().onError(obj, error);
             obj.context().get().onError(obj, error);
         }
         }
@@ -765,20 +833,66 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
 
 
     /**
     /**
      * As per the OFDPA 2.0 TTP, packets are sent out of ports by using
      * As per the OFDPA 2.0 TTP, packets are sent out of ports by using
-     * a chain of groups, namely an L3 Unicast Group that points to an L2 Interface
-     * Group which in-turn points to an output port. The Next Objective passed
+     * a chain of groups. The simple Next Objective passed
      * in by the application has to be broken up into a group chain
      * in by the application has to be broken up into a group chain
-     * to satisfy this TTP.
+     * comprising of an L3 Unicast Group that points to an L2 Interface
+     * Group which in-turn points to an output port. In some cases, the simple
+     * next Objective can just be an L2 interface without the need for chaining.
      *
      *
      * @param nextObj  the nextObjective of type SIMPLE
      * @param nextObj  the nextObjective of type SIMPLE
      */
      */
     private void processSimpleNextObjective(NextObjective nextObj) {
     private void processSimpleNextObjective(NextObjective nextObj) {
         // break up simple next objective to GroupChain objects
         // break up simple next objective to GroupChain objects
         TrafficTreatment treatment = nextObj.next().iterator().next();
         TrafficTreatment treatment = nextObj.next().iterator().next();
+
+        GroupInfo groupInfo = createL2L3Chain(treatment, nextObj.id(),
+                                              nextObj.appId(), false,
+                                              nextObj.meta());
+        if (groupInfo == null) {
+            log.error("Could not process nextObj={} in dev:{}", nextObj.id(), deviceId);
+            return;
+        }
+        // create object for local and distributed storage
+        Deque<GroupKey> gkeyChain = new ArrayDeque<>();
+        gkeyChain.addFirst(groupInfo.innerGrpDesc.appCookie());
+        gkeyChain.addFirst(groupInfo.outerGrpDesc.appCookie());
+        OfdpaNextGroup ofdpaGrp = new OfdpaNextGroup(
+                                           Collections.singletonList(gkeyChain),
+                                           nextObj);
+
+        // store l3groupkey with the ofdpaGroupChain for the nextObjective that depends on it
+        pendingNextObjectives.put(groupInfo.outerGrpDesc.appCookie(), ofdpaGrp);
+
+        // now we are ready to send the l2 groupDescription (inner), as all the stores
+        // that will get async replies have been updated. By waiting to update
+        // the stores, we prevent nasty race conditions.
+        groupService.addGroup(groupInfo.innerGrpDesc);
+    }
+
+    /**
+     * Creates one of two possible group-chains from the treatment
+     * passed in. Depending on the MPLS boolean, this method either creates
+     * an L3Unicast Group --> L2Interface Group, if mpls is false;
+     * or MPLSInterface Group --> L2Interface Group, if mpls is true;
+     * The returned 'inner' group description is always the L2 Interface group.
+     *
+     * @param treatment that needs to be broken up to create the group chain
+     * @param nextId of the next objective that needs this group chain
+     * @param appId of the application that sent this next objective
+     * @param mpls determines if L3Unicast or MPLSInterface group is created
+     * @param meta metadata passed in by the application as part of the nextObjective
+     * @return GroupInfo containing the GroupDescription of the
+     *         L2Interface group(inner) and the GroupDescription of the (outer)
+     *         L3Unicast/MPLSInterface group. May return null if there is an
+     *         error in processing the chain
+     */
+    private GroupInfo createL2L3Chain(TrafficTreatment treatment, int nextId,
+                                      ApplicationId appId, boolean mpls,
+                                      TrafficSelector meta) {
         // for the l2interface group, get vlan and port info
         // for the l2interface group, get vlan and port info
-        // for the l3unicast group, get the src/dst mac and vlan info
-        TrafficTreatment.Builder l3utt = DefaultTrafficTreatment.builder();
-        TrafficTreatment.Builder l2itt = DefaultTrafficTreatment.builder();
+        // for the outer group, get the src/dst mac, and vlan info
+        TrafficTreatment.Builder outerTtb = DefaultTrafficTreatment.builder();
+        TrafficTreatment.Builder innerTtb = DefaultTrafficTreatment.builder();
         VlanId vlanid = null;
         VlanId vlanid = null;
         long portNum = 0;
         long portNum = 0;
         for (Instruction ins : treatment.allInstructions()) {
         for (Instruction ins : treatment.allInstructions()) {
@@ -786,76 +900,144 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
                 L2ModificationInstruction l2ins = (L2ModificationInstruction) ins;
                 L2ModificationInstruction l2ins = (L2ModificationInstruction) ins;
                 switch (l2ins.subtype()) {
                 switch (l2ins.subtype()) {
                 case ETH_DST:
                 case ETH_DST:
-                    l3utt.setEthDst(((ModEtherInstruction) l2ins).mac());
+                    outerTtb.setEthDst(((ModEtherInstruction) l2ins).mac());
                     break;
                     break;
                 case ETH_SRC:
                 case ETH_SRC:
-                    l3utt.setEthSrc(((ModEtherInstruction) l2ins).mac());
+                    outerTtb.setEthSrc(((ModEtherInstruction) l2ins).mac());
                     break;
                     break;
                 case VLAN_ID:
                 case VLAN_ID:
                     vlanid = ((ModVlanIdInstruction) l2ins).vlanId();
                     vlanid = ((ModVlanIdInstruction) l2ins).vlanId();
-                    l3utt.setVlanId(vlanid);
+                    outerTtb.setVlanId(vlanid);
+                    break;
+                case VLAN_POP:
+                    innerTtb.popVlan();
                     break;
                     break;
                 case DEC_MPLS_TTL:
                 case DEC_MPLS_TTL:
                 case MPLS_LABEL:
                 case MPLS_LABEL:
                 case MPLS_POP:
                 case MPLS_POP:
                 case MPLS_PUSH:
                 case MPLS_PUSH:
                 case VLAN_PCP:
                 case VLAN_PCP:
-                case VLAN_POP:
                 case VLAN_PUSH:
                 case VLAN_PUSH:
                 default:
                 default:
                     break;
                     break;
                 }
                 }
             } else if (ins.type() == Instruction.Type.OUTPUT) {
             } else if (ins.type() == Instruction.Type.OUTPUT) {
                 portNum = ((OutputInstruction) ins).port().toLong();
                 portNum = ((OutputInstruction) ins).port().toLong();
-                l2itt.add(ins);
+                innerTtb.add(ins);
             } else {
             } else {
                 log.warn("Driver does not handle this type of TrafficTreatment"
                 log.warn("Driver does not handle this type of TrafficTreatment"
                         + " instruction in nextObjectives:  {}", ins.type());
                         + " instruction in nextObjectives:  {}", ins.type());
             }
             }
         }
         }
 
 
+        if (vlanid == null) {
+            //use the vlanid associated with the port
+            vlanid = port2Vlan.get(PortNumber.portNumber(portNum));
+        }
+
+        if (vlanid == null) {
+            // use metadata
+            for (Criterion metaCriterion : meta.criteria()) {
+                if (metaCriterion.type() == Type.VLAN_VID) {
+                    vlanid = ((VlanIdCriterion) metaCriterion).vlanId();
+                }
+            }
+        }
+
+        if (vlanid == null) {
+            log.error("Driver cannot process an L2/L3 group chain without "
+                    + "egress vlan information for dev: {} port:{}",
+                    deviceId, portNum);
+            return null;
+        }
+
         // assemble information for ofdpa l2interface group
         // assemble information for ofdpa l2interface group
-        int l2gk = nextObj.id() | GROUP1MASK;
-        final GroupKey l2groupkey = new DefaultGroupKey(appKryo.serialize(l2gk));
         Integer l2groupId = L2INTERFACEMASK | (vlanid.toShort() << 16) | (int) portNum;
         Integer l2groupId = L2INTERFACEMASK | (vlanid.toShort() << 16) | (int) portNum;
+        // a globally unique groupkey that is different for ports in the same devices
+        // but different for the same portnumber on different devices. Also different
+        // for the various group-types created out of the same next objective.
+        int l2gk = 0x0ffffff & (deviceId.hashCode() << 8 | (int) portNum);
+        final GroupKey l2groupkey = new DefaultGroupKey(appKryo.serialize(l2gk));
 
 
-        // assemble information for ofdpa l3unicast group
-        int l3gk = nextObj.id() | GROUP0MASK;
-        final GroupKey l3groupkey = new DefaultGroupKey(appKryo.serialize(l3gk));
+        // assemble information for outer group
+        GroupDescription outerGrpDesc = null;
+        if (mpls) {
+            // outer group is MPLSInteface
+            Integer mplsgroupId = MPLSINTERFACEMASK | (int) portNum;
+            // using mplsinterfacemask in groupkey to differentiate from l2interface
+            int mplsgk = MPLSINTERFACEMASK | (0x0ffffff & (deviceId.hashCode() << 8 | (int) portNum));
+            final GroupKey mplsgroupkey = new DefaultGroupKey(appKryo.serialize(mplsgk));
+            outerTtb.group(new DefaultGroupId(l2groupId));
+            // create the mpls-interface group description to wait for the
+            // l2 interface group to be processed
+            GroupBucket mplsinterfaceGroupBucket =
+                    DefaultGroupBucket.createIndirectGroupBucket(outerTtb.build());
+            outerGrpDesc = new DefaultGroupDescription(
+                                   deviceId,
+                                   GroupDescription.Type.INDIRECT,
+                                   new GroupBuckets(Collections.singletonList(
+                                                        mplsinterfaceGroupBucket)),
+                                   mplsgroupkey,
+                                   mplsgroupId,
+                                   appId);
+            log.debug("Trying MPLS-Interface: device:{} gid:{} gkey:{} nextid:{}",
+                      deviceId, Integer.toHexString(mplsgroupId),
+                      mplsgroupkey, nextId);
+        } else {
+            // outer group is L3Unicast
             Integer l3groupId = L3UNICASTMASK | (int) portNum;
             Integer l3groupId = L3UNICASTMASK | (int) portNum;
-        l3utt.group(new DefaultGroupId(l2groupId));
-        GroupChainElem gce = new GroupChainElem(l3groupkey, l3groupId,
+            int l3gk = L3UNICASTMASK | (0x0ffffff & (deviceId.hashCode() << 8 | (int) portNum));
+            final GroupKey l3groupkey = new DefaultGroupKey(appKryo.serialize(l3gk));
+            outerTtb.group(new DefaultGroupId(l2groupId));
+            // create the l3unicast group description to wait for the
+            // l2 interface group to be processed
+            GroupBucket l3unicastGroupBucket =
+                    DefaultGroupBucket.createIndirectGroupBucket(outerTtb.build());
+            outerGrpDesc = new DefaultGroupDescription(
+                                   deviceId,
                                    GroupDescription.Type.INDIRECT,
                                    GroupDescription.Type.INDIRECT,
-                                                Collections.singletonList(l3utt.build()),
-                                                nextObj.appId(), 1);
-
-        // create object for local and distributed storage
-        List<GroupKey> gkeys = new ArrayList<GroupKey>();
-        gkeys.add(l3groupkey); // group0 in chain
-        gkeys.add(l2groupkey); // group1 in chain
-        OfdpaGroupChain ofdpaGrp = new OfdpaGroupChain(gkeys, nextObj);
-
-        // store l2groupkey with the groupChainElem for the l3group that depends on it
-        pendingGroups.put(l2groupkey, gce);
-
-        // store l3groupkey with the ofdpaGroupChain for the nextObjective that depends on it
-        pendingNextObjectives.put(l3groupkey, ofdpaGrp);
-
-        // create group description for the ofdpa l2interfacegroup and send to groupservice
-        GroupBucket bucket =
-                DefaultGroupBucket.createIndirectGroupBucket(l2itt.build());
-        GroupDescription groupDescription = new DefaultGroupDescription(deviceId,
+                                   new GroupBuckets(Collections.singletonList(
+                                                        l3unicastGroupBucket)),
+                                   l3groupkey,
+                                   l3groupId,
+                                   appId);
+            log.debug("Trying L3Unicast: device:{} gid:{} gkey:{} nextid:{}",
+                      deviceId, Integer.toHexString(l3groupId),
+                      l3groupkey, nextId);
+        }
+
+        // store l2groupkey with the groupChainElem for the outer-group that depends on it
+        GroupChainElem gce = new GroupChainElem(outerGrpDesc, 1);
+        Set<GroupChainElem> gceSet = Collections.newSetFromMap(
+                                         new ConcurrentHashMap<GroupChainElem, Boolean>());
+        gceSet.add(gce);
+        Set<GroupChainElem> retval = pendingGroups.putIfAbsent(l2groupkey, gceSet);
+        if (retval != null) {
+            retval.add(gce);
+        }
+
+        // create group description for the inner l2interfacegroup
+        GroupBucket l2interfaceGroupBucket =
+                DefaultGroupBucket.createIndirectGroupBucket(innerTtb.build());
+        GroupDescription l2groupDescription =
+                             new DefaultGroupDescription(
+                                     deviceId,
                                      GroupDescription.Type.INDIRECT,
                                      GroupDescription.Type.INDIRECT,
-                             new GroupBuckets(Collections.singletonList(bucket)),
+                                     new GroupBuckets(Collections.singletonList(
+                                                          l2interfaceGroupBucket)),
                                      l2groupkey,
                                      l2groupkey,
                                      l2groupId,
                                      l2groupId,
-                             nextObj.appId());
-        groupService.addGroup(groupDescription);
+                                     appId);
+        log.debug("Trying L2Interface: device:{} gid:{} gkey:{} nextId:{}",
+                  deviceId, Integer.toHexString(l2groupId),
+                  l2groupkey, nextId);
+        return new GroupInfo(l2groupDescription, outerGrpDesc);
+
     }
     }
 
 
     /**
     /**
      * As per the OFDPA 2.0 TTP, packets are sent out of ports by using
      * As per the OFDPA 2.0 TTP, packets are sent out of ports by using
-     * a chain of groups. The Next Objective passed in by the application
+     * a chain of groups. The broadcast Next Objective passed in by the application
      * has to be broken up into a group chain comprising of an
      * has to be broken up into a group chain comprising of an
      * L2 Flood group whose buckets point to L2 Interface groups.
      * L2 Flood group whose buckets point to L2 Interface groups.
      *
      *
@@ -866,9 +1048,9 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
         Collection<TrafficTreatment> buckets = nextObj.next();
         Collection<TrafficTreatment> buckets = nextObj.next();
 
 
         // each treatment is converted to an L2 interface group
         // each treatment is converted to an L2 interface group
-        int indicator = 0;
         VlanId vlanid = null;
         VlanId vlanid = null;
-        List<GroupInfo> groupInfoCollection = new ArrayList<>();
+        List<GroupDescription> l2interfaceGroupDescs = new ArrayList<>();
+        List<Deque<GroupKey>> allGroupKeys = new ArrayList<>();
         for (TrafficTreatment treatment : buckets) {
         for (TrafficTreatment treatment : buckets) {
             TrafficTreatment.Builder newTreatment = DefaultTrafficTreatment.builder();
             TrafficTreatment.Builder newTreatment = DefaultTrafficTreatment.builder();
             PortNumber portNum = null;
             PortNumber portNum = null;
@@ -907,83 +1089,284 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
                 }
                 }
             }
             }
 
 
-            // assemble info for all l2 interface groups
-            indicator += GROUP1MASK;
-            int l2gk = nextObj.id() | indicator;
+            // assemble info for l2 interface group
+            int l2gk = 0x0ffffff & (deviceId.hashCode() << 8 | (int) portNum.toLong());
             final GroupKey l2groupkey = new DefaultGroupKey(appKryo.serialize(l2gk));
             final GroupKey l2groupkey = new DefaultGroupKey(appKryo.serialize(l2gk));
             Integer l2groupId = L2INTERFACEMASK | (vlanid.toShort() << 16) |
             Integer l2groupId = L2INTERFACEMASK | (vlanid.toShort() << 16) |
                                     (int) portNum.toLong();
                                     (int) portNum.toLong();
-            GroupBucket newbucket =
+            GroupBucket l2interfaceGroupBucket =
                     DefaultGroupBucket.createIndirectGroupBucket(newTreatment.build());
                     DefaultGroupBucket.createIndirectGroupBucket(newTreatment.build());
+            GroupDescription l2interfaceGroupDescription =
+                        new DefaultGroupDescription(
+                                                    deviceId,
+                            GroupDescription.Type.INDIRECT,
+                            new GroupBuckets(Collections.singletonList(
+                                                 l2interfaceGroupBucket)),
+                            l2groupkey,
+                            l2groupId,
+                            nextObj.appId());
+            log.debug("Trying L2-Interface: device:{} gid:{} gkey:{} nextid:{}",
+                      deviceId, Integer.toHexString(l2groupId),
+                      l2groupkey, nextObj.id());
+
+            Deque<GroupKey> gkeyChain = new ArrayDeque<>();
+            gkeyChain.addFirst(l2groupkey);
 
 
             // store the info needed to create this group
             // store the info needed to create this group
-            groupInfoCollection.add(new GroupInfo(l2groupId, l2groupkey, newbucket));
+            l2interfaceGroupDescs.add(l2interfaceGroupDescription);
+            allGroupKeys.add(gkeyChain);
         }
         }
 
 
         // assemble info for l2 flood group
         // assemble info for l2 flood group
-        int l2floodgk = nextObj.id() | GROUP0MASK;
-        final GroupKey l2floodgroupkey = new DefaultGroupKey(appKryo.serialize(l2floodgk));
         Integer l2floodgroupId = L2FLOODMASK | (vlanid.toShort() << 16) | nextObj.id();
         Integer l2floodgroupId = L2FLOODMASK | (vlanid.toShort() << 16) | nextObj.id();
-        // collection of treatment with groupids of l2 interface groups
-        List<TrafficTreatment> floodtt = new ArrayList<>();
-        for (GroupInfo gi : groupInfoCollection) {
+        int l2floodgk = L2FLOODMASK | nextObj.id() << 12;
+        final GroupKey l2floodgroupkey = new DefaultGroupKey(appKryo.serialize(l2floodgk));
+        // collection of group buckets pointing to all the l2 interface groups
+        List<GroupBucket> l2floodBuckets = new ArrayList<>();
+        for (GroupDescription l2intGrpDesc : l2interfaceGroupDescs) {
             TrafficTreatment.Builder ttb = DefaultTrafficTreatment.builder();
             TrafficTreatment.Builder ttb = DefaultTrafficTreatment.builder();
-            ttb.group(new DefaultGroupId(gi.groupId));
-            floodtt.add(ttb.build());
+            ttb.group(new DefaultGroupId(l2intGrpDesc.givenGroupId()));
+            GroupBucket abucket = DefaultGroupBucket.createAllGroupBucket(ttb.build());
+            l2floodBuckets.add(abucket);
         }
         }
-        GroupChainElem gce = new GroupChainElem(l2floodgroupkey, l2floodgroupId,
+        // create the l2flood group-description to wait for all the
+        // l2interface groups to be processed
+        GroupDescription l2floodGroupDescription =
+                                new DefaultGroupDescription(
+                                        deviceId,
                                         GroupDescription.Type.ALL,
                                         GroupDescription.Type.ALL,
-                                                floodtt,
-                                                nextObj.appId(),
-                                                groupInfoCollection.size());
+                                        new GroupBuckets(l2floodBuckets),
+                                        l2floodgroupkey,
+                                        l2floodgroupId,
+                                        nextObj.appId());
+        GroupChainElem gce = new GroupChainElem(l2floodGroupDescription,
+                                                l2interfaceGroupDescs.size());
+        log.debug("Trying L2-Flood: device:{} gid:{} gkey:{} nextid:{}",
+                  deviceId, Integer.toHexString(l2floodgroupId),
+                  l2floodgroupkey, nextObj.id());
 
 
         // create objects for local and distributed storage
         // create objects for local and distributed storage
-        List<GroupKey> gkeys = new ArrayList<GroupKey>();
-        gkeys.add(l2floodgroupkey); // group0 in chain
-        OfdpaGroupChain ofdpaGrp = new OfdpaGroupChain(gkeys, nextObj);
+        allGroupKeys.forEach(gkeyChain -> gkeyChain.addFirst(l2floodgroupkey));
+        OfdpaNextGroup ofdpaGrp = new OfdpaNextGroup(allGroupKeys, nextObj);
 
 
         // store l2floodgroupkey with the ofdpaGroupChain for the nextObjective
         // store l2floodgroupkey with the ofdpaGroupChain for the nextObjective
         // that depends on it
         // that depends on it
         pendingNextObjectives.put(l2floodgroupkey, ofdpaGrp);
         pendingNextObjectives.put(l2floodgroupkey, ofdpaGrp);
 
 
-        for (GroupInfo gi : groupInfoCollection) {
+        for (GroupDescription l2intGrpDesc : l2interfaceGroupDescs) {
             // store all l2groupkeys with the groupChainElem for the l2floodgroup
             // store all l2groupkeys with the groupChainElem for the l2floodgroup
             // that depends on it
             // that depends on it
-            pendingGroups.put(gi.groupKey, gce);
+            Set<GroupChainElem> gceSet = Collections.newSetFromMap(
+                                             new ConcurrentHashMap<GroupChainElem, Boolean>());
+            gceSet.add(gce);
+            Set<GroupChainElem> retval = pendingGroups.putIfAbsent(
+                                             l2intGrpDesc.appCookie(), gceSet);
+            if (retval != null) {
+                retval.add(gce);
+            }
 
 
             // create and send groups for all l2 interface groups
             // create and send groups for all l2 interface groups
-            GroupDescription groupDescription =
+            groupService.addGroup(l2intGrpDesc);
+        }
+    }
+
+    /**
+     * Utility class for moving group information around.
+     *
+     */
+    private class GroupInfo {
+        private GroupDescription innerGrpDesc;
+        private GroupDescription outerGrpDesc;
+
+        GroupInfo(GroupDescription innerGrpDesc, GroupDescription outerGrpDesc) {
+            this.innerGrpDesc = innerGrpDesc;
+            this.outerGrpDesc = outerGrpDesc;
+        }
+    }
+
+    /**
+     * As per the OFDPA 2.0 TTP, packets are sent out of ports by using
+     * a chain of groups. The hashed Next Objective passed in by the application
+     * has to be broken up into a group chain comprising of an
+     * L3 ECMP group as the top level group. Buckets of this group can point
+     * to a variety of groups in a group chain, depending on the whether
+     * MPLS labels are being pushed or not.
+     * <p>
+     * NOTE: We do not create MPLS ECMP groups as they are unimplemented in
+     *       OF-DPA 2.0 (even though it is in the spec). Therefore we do not
+     *       check the nextObjective meta.
+     *
+     * @param nextObj  the nextObjective of type HASHED
+     */
+    private void processHashedNextObjective(NextObjective nextObj) {
+        // break up hashed next objective to multiple groups
+        Collection<TrafficTreatment> buckets = nextObj.next();
+
+        // storage for all group keys in the chain of groups created
+        List<Deque<GroupKey>> allGroupKeys = new ArrayList<>();
+        List<GroupInfo> unsentGroups = new ArrayList<>();
+        for (TrafficTreatment bucket : buckets) {
+            //figure out how many labels are pushed in each bucket
+            int labelsPushed = 0;
+            MplsLabel innermostLabel = null;
+            for (Instruction ins : bucket.allInstructions()) {
+                if (ins.type() == Instruction.Type.L2MODIFICATION) {
+                    L2ModificationInstruction l2ins = (L2ModificationInstruction) ins;
+                    if (l2ins.subtype() == L2SubType.MPLS_PUSH) {
+                        labelsPushed++;
+                    }
+                    if (l2ins.subtype() == L2SubType.MPLS_LABEL) {
+                        if (innermostLabel == null) {
+                            innermostLabel = ((ModMplsLabelInstruction) l2ins).mplsLabel();
+                        }
+                    }
+                }
+            }
+
+            Deque<GroupKey> gkeyChain = new ArrayDeque<>();
+            // XXX we only deal with 0 and 1 label push right now
+            if (labelsPushed == 0) {
+                GroupInfo nolabelGroupInfo = createL2L3Chain(bucket, nextObj.id(),
+                                                             nextObj.appId(), false,
+                                                             nextObj.meta());
+                if (nolabelGroupInfo == null) {
+                    log.error("Could not process nextObj={} in dev:{}",
+                              nextObj.id(), deviceId);
+                    return;
+                }
+                gkeyChain.addFirst(nolabelGroupInfo.innerGrpDesc.appCookie());
+                gkeyChain.addFirst(nolabelGroupInfo.outerGrpDesc.appCookie());
+
+                // we can't send the inner group description yet, as we have to
+                // create the dependent ECMP group first. So we store..
+                unsentGroups.add(nolabelGroupInfo);
+
+            } else if (labelsPushed == 1) {
+                GroupInfo onelabelGroupInfo = createL2L3Chain(bucket, nextObj.id(),
+                                                              nextObj.appId(), true,
+                                                              nextObj.meta());
+                if (onelabelGroupInfo == null) {
+                    log.error("Could not process nextObj={} in dev:{}",
+                              nextObj.id(), deviceId);
+                    return;
+                }
+                // we need to add another group to this chain - the L3VPN group
+                TrafficTreatment.Builder l3vpnTtb = DefaultTrafficTreatment.builder();
+                l3vpnTtb.pushMpls()
+                            .setMpls(innermostLabel)
+                            .setMplsBos(true)
+                            .copyTtlOut()
+                            .group(new DefaultGroupId(
+                                 onelabelGroupInfo.outerGrpDesc.givenGroupId()));
+                GroupBucket l3vpnGrpBkt  =
+                        DefaultGroupBucket.createIndirectGroupBucket(l3vpnTtb.build());
+                int l3vpngroupId = L3VPNMASK | l3vpnindex.incrementAndGet();
+                int l3vpngk = L3VPNMASK | nextObj.id() << 12 | l3vpnindex.get();
+                GroupKey l3vpngroupkey = new DefaultGroupKey(appKryo.serialize(l3vpngk));
+                GroupDescription l3vpnGroupDesc =
                         new DefaultGroupDescription(
                         new DefaultGroupDescription(
                                 deviceId,
                                 deviceId,
                                 GroupDescription.Type.INDIRECT,
                                 GroupDescription.Type.INDIRECT,
-                            new GroupBuckets(Collections.singletonList(gi.groupBucket)),
-                            gi.groupKey,
-                            gi.groupId,
+                                new GroupBuckets(Collections.singletonList(
+                                                     l3vpnGrpBkt)),
+                                l3vpngroupkey,
+                                l3vpngroupId,
                                 nextObj.appId());
                                 nextObj.appId());
-            groupService.addGroup(groupDescription);
+                GroupChainElem l3vpnGce = new GroupChainElem(l3vpnGroupDesc, 1);
+                Set<GroupChainElem> gceSet = Collections.newSetFromMap(
+                                                 new ConcurrentHashMap<GroupChainElem, Boolean>());
+                gceSet.add(l3vpnGce);
+                Set<GroupChainElem> retval = pendingGroups
+                        .putIfAbsent(onelabelGroupInfo.outerGrpDesc.appCookie(), gceSet);
+                if (retval != null) {
+                    retval.add(l3vpnGce);
                 }
                 }
+
+                gkeyChain.addFirst(onelabelGroupInfo.innerGrpDesc.appCookie());
+                gkeyChain.addFirst(onelabelGroupInfo.outerGrpDesc.appCookie());
+                gkeyChain.addFirst(l3vpngroupkey);
+
+                //now we can replace the outerGrpDesc with the one we just created
+                onelabelGroupInfo.outerGrpDesc = l3vpnGroupDesc;
+
+                // we can't send the innermost group yet, as we have to create
+                // the dependent ECMP group first. So we store ...
+                unsentGroups.add(onelabelGroupInfo);
+
+                log.debug("Trying L3VPN: device:{} gid:{} gkey:{} nextId:{}",
+                          deviceId, Integer.toHexString(l3vpngroupId),
+                          l3vpngroupkey, nextObj.id());
+
+            } else {
+                log.warn("Driver currently does not handle more than 1 MPLS "
+                        + "labels. Not processing nextObjective {}", nextObj);
+                return;
             }
             }
 
 
-    private class GroupInfo {
-        private Integer groupId;
-        private GroupKey groupKey;
-        private GroupBucket groupBucket;
+            // all groups in this chain
+            allGroupKeys.add(gkeyChain);
+        }
 
 
-        GroupInfo(Integer groupId, GroupKey groupKey, GroupBucket groupBucket) {
-            this.groupBucket = groupBucket;
-            this.groupId = groupId;
-            this.groupKey = groupKey;
+        // now we can create the outermost L3 ECMP group
+        List<GroupBucket> l3ecmpGroupBuckets = new ArrayList<>();
+        for (GroupInfo gi : unsentGroups) {
+            // create ECMP bucket to point to the outer group
+            TrafficTreatment.Builder ttb = DefaultTrafficTreatment.builder();
+            ttb.group(new DefaultGroupId(gi.outerGrpDesc.givenGroupId()));
+            GroupBucket sbucket = DefaultGroupBucket
+                    .createSelectGroupBucket(ttb.build());
+            l3ecmpGroupBuckets.add(sbucket);
         }
         }
+        int l3ecmpGroupId = L3ECMPMASK | nextObj.id() << 12;
+        GroupKey l3ecmpGroupKey = new DefaultGroupKey(appKryo.serialize(l3ecmpGroupId));
+        GroupDescription l3ecmpGroupDesc =
+                new DefaultGroupDescription(
+                        deviceId,
+                        GroupDescription.Type.SELECT,
+                        new GroupBuckets(l3ecmpGroupBuckets),
+                        l3ecmpGroupKey,
+                        l3ecmpGroupId,
+                        nextObj.appId());
+        GroupChainElem l3ecmpGce = new GroupChainElem(l3ecmpGroupDesc,
+                                                      l3ecmpGroupBuckets.size());
+
+        // create objects for local and distributed storage
+        allGroupKeys.forEach(gkeyChain -> gkeyChain.addFirst(l3ecmpGroupKey));
+        OfdpaNextGroup ofdpaGrp = new OfdpaNextGroup(allGroupKeys, nextObj);
+
+        // store l3ecmpGroupKey with the ofdpaGroupChain for the nextObjective
+        // that depends on it
+        pendingNextObjectives.put(l3ecmpGroupKey, ofdpaGrp);
+
+        log.debug("Trying L3ECMP: device:{} gid:{} gkey:{} nextId:{}",
+                  deviceId, Integer.toHexString(l3ecmpGroupId),
+                  l3ecmpGroupKey, nextObj.id());
+        // finally we are ready to send the innermost groups
+        for (GroupInfo gi : unsentGroups) {
+            log.debug("Sending innermost group {} in group chain on device {} ",
+                      Integer.toHexString(gi.innerGrpDesc.givenGroupId()), deviceId);
+            Set<GroupChainElem> gceSet = Collections.newSetFromMap(
+                                             new ConcurrentHashMap<GroupChainElem, Boolean>());
+            gceSet.add(l3ecmpGce);
+            Set<GroupChainElem> retval = pendingGroups
+                    .putIfAbsent(gi.outerGrpDesc.appCookie(), gceSet);
+            if (retval != null) {
+                retval.add(l3ecmpGce);
+            }
+
+            groupService.addGroup(gi.innerGrpDesc);
         }
         }
 
 
-    private void processHashedNextObjective(NextObjective nextObj) {
-        // TODO Auto-generated method stub
     }
     }
 
 
     private void addBucketToGroup(NextObjective nextObjective) {
     private void addBucketToGroup(NextObjective nextObjective) {
         // TODO Auto-generated method stub
         // TODO Auto-generated method stub
     }
     }
 
 
+    private void waitToAddBucketToGroup(NextObjective nextObjective) {
+        // TODO Auto-generated method stub
+    }
+
     private void removeBucketFromGroup(NextObjective nextObjective) {
     private void removeBucketFromGroup(NextObjective nextObjective) {
         // TODO Auto-generated method stub
         // TODO Auto-generated method stub
     }
     }
@@ -1009,45 +1392,11 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
     private void processGroupChain(GroupChainElem gce) {
     private void processGroupChain(GroupChainElem gce) {
         int waitOnGroups = gce.decrementAndGetGroupsWaitedOn();
         int waitOnGroups = gce.decrementAndGetGroupsWaitedOn();
         if (waitOnGroups != 0) {
         if (waitOnGroups != 0) {
-            log.debug("GCE: {} waiting on {} groups. Not processing yet",
-                      gce, waitOnGroups);
+            log.debug("GCE: {} not ready to be processed", gce);
             return;
             return;
         }
         }
-        List<GroupBucket> buckets = new ArrayList<>();
-        switch (gce.groupType) {
-        case INDIRECT:
-            GroupBucket ibucket = DefaultGroupBucket
-                .createIndirectGroupBucket(gce.bucketActions.iterator().next());
-            buckets.add(ibucket);
-            break;
-        case ALL:
-            for (TrafficTreatment tt : gce.bucketActions) {
-                GroupBucket abucket = DefaultGroupBucket
-                        .createAllGroupBucket(tt);
-                buckets.add(abucket);
-            }
-            break;
-        case SELECT:
-            for (TrafficTreatment tt : gce.bucketActions) {
-                GroupBucket sbucket = DefaultGroupBucket
-                        .createSelectGroupBucket(tt);
-                buckets.add(sbucket);
-            }
-            break;
-        case FAILOVER:
-        default:
-            log.error("Unknown or unimplemented GroupChainElem {}", gce);
-        }
-
-        if (buckets.size() > 0) {
-            GroupDescription groupDesc = new DefaultGroupDescription(
-                                                 deviceId, gce.groupType,
-                                                 new GroupBuckets(buckets),
-                                                 gce.gkey,
-                                                 gce.givenGroupId,
-                                                 gce.appId);
-            groupService.addGroup(groupDesc);
-        }
+        log.debug("GCE: {} ready to be processed", gce);
+        groupService.addGroup(gce.groupDescription);
     }
     }
 
 
     private class GroupChecker implements Runnable {
     private class GroupChecker implements Runnable {
@@ -1063,19 +1412,23 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
 
 
             keys.stream().forEach(key -> {
             keys.stream().forEach(key -> {
                 //first check for group chain
                 //first check for group chain
-                GroupChainElem gce = pendingGroups.remove(key);
-                if (gce != null) {
-                    log.info("Group service processed group key {}. Processing next "
-                            + "group in group chain with group key {}",
-                            appKryo.deserialize(key.key()),
-                            appKryo.deserialize(gce.gkey.key()));
+                Set<GroupChainElem> gceSet = pendingGroups.remove(key);
+                if (gceSet != null) {
+                    for (GroupChainElem gce : gceSet) {
+                        log.info("Group service processed group key {} in device {}. "
+                                + "Processing next group in group chain with group id {}",
+                                key, deviceId,
+                                Integer.toHexString(gce.groupDescription.givenGroupId()));
                         processGroupChain(gce);
                         processGroupChain(gce);
+                    }
                 } else {
                 } else {
-                    OfdpaGroupChain obj = pendingNextObjectives.getIfPresent(key);
-                    log.info("Group service processed group key {}. Done implementing "
-                            + "next objective: {}", appKryo.deserialize(key.key()),
-                            obj.nextObjective().id());
+                    OfdpaNextGroup obj = pendingNextObjectives.getIfPresent(key);
                     if (obj != null) {
                     if (obj != null) {
+                        log.info("Group service processed group key {} in device:{}. "
+                                + "Done implementing next objective: {} <<-->> gid:{}",
+                                key, deviceId, obj.nextObjective().id(),
+                                Integer.toHexString(groupService.getGroup(deviceId, key)
+                                                    .givenGroupId()));
                         pass(obj.nextObjective());
                         pass(obj.nextObjective());
                         pendingNextObjectives.invalidate(key);
                         pendingNextObjectives.invalidate(key);
                         flowObjectiveStore.putNextGroup(obj.nextObjective().id(), obj);
                         flowObjectiveStore.putNextGroup(obj.nextObjective().id(), obj);
@@ -1088,23 +1441,27 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
     private class InnerGroupListener implements GroupListener {
     private class InnerGroupListener implements GroupListener {
         @Override
         @Override
         public void event(GroupEvent event) {
         public void event(GroupEvent event) {
-            log.debug("received group event of type {}", event.type());
+            log.trace("received group event of type {}", event.type());
             if (event.type() == GroupEvent.Type.GROUP_ADDED) {
             if (event.type() == GroupEvent.Type.GROUP_ADDED) {
                 GroupKey key = event.subject().appCookie();
                 GroupKey key = event.subject().appCookie();
                 // first check for group chain
                 // first check for group chain
-                GroupChainElem gce = pendingGroups.remove(key);
-                if (gce != null) {
+                Set<GroupChainElem> gceSet = pendingGroups.remove(key);
+                if (gceSet != null) {
+                    for (GroupChainElem gce : gceSet) {
                         log.info("group ADDED with group key {} .. "
                         log.info("group ADDED with group key {} .. "
                                 + "Processing next group in group chain with group key {}",
                                 + "Processing next group in group chain with group key {}",
-                            appKryo.deserialize(key.key()),
-                            appKryo.deserialize(gce.gkey.key()));
+                                key,
+                                gce.groupDescription.appCookie());
                         processGroupChain(gce);
                         processGroupChain(gce);
+                    }
                 } else {
                 } else {
-                    OfdpaGroupChain obj = pendingNextObjectives.getIfPresent(key);
+                    OfdpaNextGroup obj = pendingNextObjectives.getIfPresent(key);
                     if (obj != null) {
                     if (obj != null) {
-                        log.info("group ADDED with key {}.. Done implementing next "
-                                + "objective: {}",
-                                appKryo.deserialize(key.key()), obj.nextObjective().id());
+                        log.info("group ADDED with key {} in dev {}.. Done implementing next "
+                                + "objective: {} <<-->> gid:{}",
+                                key, deviceId, obj.nextObjective().id(),
+                                Integer.toHexString(groupService.getGroup(deviceId, key)
+                                                    .givenGroupId()));
                         pass(obj.nextObjective());
                         pass(obj.nextObjective());
                         pendingNextObjectives.invalidate(key);
                         pendingNextObjectives.invalidate(key);
                         flowObjectiveStore.putNextGroup(obj.nextObjective().id(), obj);
                         flowObjectiveStore.putNextGroup(obj.nextObjective().id(), obj);
@@ -1115,30 +1472,35 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
     }
     }
 
 
     /**
     /**
-     * Represents a group-chain that implements a Next-Objective from
-     * the application. Includes information about the next objective Id, and the
-     * group keys for the groups in the group chain. The chain is expected to
-     * look like group0 --> group 1 --> outPort. Information about the groups
-     * themselves can be fetched from the Group Service using the group keys from
-     * objects instantiating this class.
+     * Represents an entire group-chain that implements a Next-Objective from
+     * the application. The objective is represented as a list of deques, where
+     * each deque can is a separate chain of groups.
+     * <p>
+     * For example, an ECMP group with 3 buckets, where each bucket points to
+     * a group chain of L3 Unicast and L2 interface groups will look like this:
+     * <ul>
+     * <li>List[0] is a Deque of GroupKeyECMP(first)-GroupKeyL3(middle)-GroupKeyL2(last)
+     * <li>List[1] is a Deque of GroupKeyECMP(first)-GroupKeyL3(middle)-GroupKeyL2(last)
+     * <li>List[2] is a Deque of GroupKeyECMP(first)-GroupKeyL3(middle)-GroupKeyL2(last)
+     * </ul>
+     * where the first element of each deque is the same, representing the
+     * top level ECMP group, while every other element represents a unique groupKey.
+     * <p>
+     * Also includes information about the next objective that
+     * resulted in this group-chain.
      *
      *
-     * XXX Revisit this - since the forwarding objective only ever needs the
-     * groupkey of the top-level group in the group chain, why store a series
-     * of groupkeys. Also the group-chain list only works for 1-to-1 chaining,
-     * not for 1-to-many chaining.
      */
      */
-    private class OfdpaGroupChain implements NextGroup {
+    private class OfdpaNextGroup implements NextGroup {
         private final NextObjective nextObj;
         private final NextObjective nextObj;
-        private final List<GroupKey> gkeys;
+        private final List<Deque<GroupKey>> gkeys;
 
 
-        /** expected group chain: group0 --> group1 --> port. */
-        public OfdpaGroupChain(List<GroupKey> gkeys, NextObjective nextObj) {
+        public OfdpaNextGroup(List<Deque<GroupKey>> gkeys, NextObjective nextObj) {
             this.gkeys = gkeys;
             this.gkeys = gkeys;
             this.nextObj = nextObj;
             this.nextObj = nextObj;
         }
         }
 
 
         @SuppressWarnings("unused")
         @SuppressWarnings("unused")
-        public List<GroupKey> groupKeys() {
+        public List<Deque<GroupKey>> groupKey() {
             return gkeys;
             return gkeys;
         }
         }
 
 
@@ -1161,22 +1523,11 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
      * preceding groups in the group chain to be created.
      * preceding groups in the group chain to be created.
      */
      */
     private class GroupChainElem {
     private class GroupChainElem {
-        private Collection<TrafficTreatment> bucketActions;
-        private Integer givenGroupId;
-        private GroupDescription.Type groupType;
-        private GroupKey gkey;
-        private ApplicationId appId;
+        private GroupDescription groupDescription;
         private AtomicInteger waitOnGroups;
         private AtomicInteger waitOnGroups;
 
 
-        GroupChainElem(GroupKey gkey, Integer givenGroupId,
-                       GroupDescription.Type groupType,
-                       Collection<TrafficTreatment> tr, ApplicationId appId,
-                       int waitOnGroups) {
-            this.bucketActions = tr;
-            this.givenGroupId = givenGroupId;
-            this.groupType = groupType;
-            this.gkey = gkey;
-            this.appId = appId;
+        GroupChainElem(GroupDescription groupDescription, int waitOnGroups) {
+            this.groupDescription = groupDescription;
             this.waitOnGroups = new AtomicInteger(waitOnGroups);
             this.waitOnGroups = new AtomicInteger(waitOnGroups);
         }
         }
 
 
@@ -1194,7 +1545,10 @@ public class OFDPA2Pipeline extends AbstractHandlerBehaviour implements Pipeline
 
 
         @Override
         @Override
         public String toString() {
         public String toString() {
-            return Integer.toHexString(givenGroupId);
+            return (Integer.toHexString(groupDescription.givenGroupId()) +
+                    " groupKey: " + groupDescription.appCookie() +
+                    " waiting-on-groups: " + waitOnGroups.get() +
+                    " device: " + deviceId);
         }
         }
 
 
     }
     }

@@ -47,6 +47,7 @@ import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.criteria.Criteria;
 import org.onosproject.net.flow.criteria.Criteria;
 import org.onosproject.net.flow.criteria.Criterion;
 import org.onosproject.net.flow.criteria.Criterion;
+import org.onosproject.net.flow.criteria.Criterion.Type;
 import org.onosproject.net.flow.criteria.EthCriterion;
 import org.onosproject.net.flow.criteria.EthCriterion;
 import org.onosproject.net.flow.criteria.EthTypeCriterion;
 import org.onosproject.net.flow.criteria.EthTypeCriterion;
 import org.onosproject.net.flow.criteria.IPCriterion;
 import org.onosproject.net.flow.criteria.IPCriterion;
@@ -73,6 +74,7 @@ import org.onosproject.net.group.GroupEvent;
 import org.onosproject.net.group.GroupKey;
 import org.onosproject.net.group.GroupKey;
 import org.onosproject.net.group.GroupListener;
 import org.onosproject.net.group.GroupListener;
 import org.onosproject.net.group.GroupService;
 import org.onosproject.net.group.GroupService;
+import org.onosproject.store.serializers.KryoNamespaces;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
 import java.util.ArrayList;
 import java.util.ArrayList;
@@ -129,8 +131,13 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                                     groupedThreads("onos/pipeliner",
                                     groupedThreads("onos/pipeliner",
                                                    "spring-open-%d"));
                                                    "spring-open-%d"));
     protected KryoNamespace appKryo = new KryoNamespace.Builder()
     protected KryoNamespace appKryo = new KryoNamespace.Builder()
-            .register(GroupKey.class).register(DefaultGroupKey.class)
-            .register(SegmentRoutingGroup.class).register(byte[].class).build();
+            .register(KryoNamespaces.API)
+            .register(GroupKey.class)
+            .register(DefaultGroupKey.class)
+            .register(TrafficTreatment.class)
+            .register(SpringOpenGroup.class)
+            .register(byte[].class)
+            .build();
 
 
     @Override
     @Override
     public void init(DeviceId deviceId, PipelinerContext context) {
     public void init(DeviceId deviceId, PipelinerContext context) {
@@ -202,17 +209,15 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                     @Override
                     @Override
                     public void onSuccess(FlowRuleOperations ops) {
                     public void onSuccess(FlowRuleOperations ops) {
                         pass(fwd);
                         pass(fwd);
-                        log.debug("Provisioned tables in {} with "
-                                + "forwarding rules for segment "
-                                + "router", deviceId);
+                        log.debug("Provisioned tables in {} successfully with "
+                                + "forwarding rules", deviceId);
                     }
                     }
 
 
                     @Override
                     @Override
                     public void onError(FlowRuleOperations ops) {
                     public void onError(FlowRuleOperations ops) {
                         fail(fwd, ObjectiveError.FLOWINSTALLATIONFAILED);
                         fail(fwd, ObjectiveError.FLOWINSTALLATIONFAILED);
                         log.warn("Failed to provision tables in {} with "
                         log.warn("Failed to provision tables in {} with "
-                                + "forwarding rules for segment router",
-                                deviceId);
+                                + "forwarding rules", deviceId);
                     }
                     }
                 }));
                 }));
 
 
@@ -220,26 +225,50 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
 
 
     @Override
     @Override
     public void next(NextObjective nextObjective) {
     public void next(NextObjective nextObjective) {
-
-        log.debug("Processing NextObjective id{} op{}", nextObjective.id(),
-                  nextObjective.op());
-        if (nextObjective.op() == Objective.Operation.REMOVE) {
-            if (nextObjective.next().isEmpty()) {
-                removeGroup(nextObjective);
-            } else {
-                removeBucketFromGroup(nextObjective);
-            }
-        } else if (nextObjective.op() == Objective.Operation.ADD) {
         NextGroup nextGroup = flowObjectiveStore.getNextGroup(nextObjective.id());
         NextGroup nextGroup = flowObjectiveStore.getNextGroup(nextObjective.id());
+        switch (nextObjective.op()) {
+        case ADD:
             if (nextGroup != null) {
             if (nextGroup != null) {
+                log.warn("Cannot add next {} that already exists in device {}",
+                         nextObjective.id(), deviceId);
+                return;
+            }
+            log.debug("Processing NextObjective id{} in dev{} - add group",
+                      nextObjective.id(), deviceId);
+            addGroup(nextObjective);
+            break;
+        case ADD_TO_EXISTING:
+            if (nextGroup != null) {
+                log.debug("Processing NextObjective id{} in dev{} - add bucket",
+                          nextObjective.id(), deviceId);
                 addBucketToGroup(nextObjective);
                 addBucketToGroup(nextObjective);
             } else {
             } else {
-                addGroup(nextObjective);
+                log.warn("Cannot add to group that does not exist");
             }
             }
-        } else {
+            break;
+        case REMOVE:
+            if (nextGroup == null) {
+                log.warn("Cannot remove next {} that does not exist in device {}",
+                         nextObjective.id(), deviceId);
+                return;
+            }
+            log.debug("Processing NextObjective id{}  in dev{} - remove group",
+                      nextObjective.id(), deviceId);
+            removeGroup(nextObjective);
+            break;
+        case REMOVE_FROM_EXISTING:
+            if (nextGroup == null) {
+                log.warn("Cannot remove from next {} that does not exist in device {}",
+                         nextObjective.id(), deviceId);
+                return;
+            }
+            log.debug("Processing NextObjective id{} in dev{} - remove bucket",
+                      nextObjective.id(), deviceId);
+            removeBucketFromGroup(nextObjective);
+            break;
+        default:
             log.warn("Unsupported operation {}", nextObjective.op());
             log.warn("Unsupported operation {}", nextObjective.op());
         }
         }
-
     }
     }
 
 
     private void removeGroup(NextObjective nextObjective) {
     private void removeGroup(NextObjective nextObjective) {
@@ -256,7 +285,6 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
         List<GroupBucket> buckets;
         List<GroupBucket> buckets;
         switch (nextObjective.type()) {
         switch (nextObjective.type()) {
             case SIMPLE:
             case SIMPLE:
-                log.debug("processing SIMPLE next objective");
                 Collection<TrafficTreatment> treatments = nextObjective.next();
                 Collection<TrafficTreatment> treatments = nextObjective.next();
                 if (treatments.size() == 1) {
                 if (treatments.size() == 1) {
                     TrafficTreatment treatment = treatments.iterator().next();
                     TrafficTreatment treatment = treatments.iterator().next();
@@ -273,14 +301,33 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                             key,
                             key,
                             null,
                             null,
                             nextObjective.appId());
                             nextObjective.appId());
-                    log.debug("Creating SIMPLE group for next objective id {}",
-                              nextObjective.id());
-                    groupService.addGroup(groupDescription);
+                    log.debug("Creating SIMPLE group for next objective id {} "
+                            + "in dev:{}", nextObjective.id(), deviceId);
                     pendingGroups.put(key, nextObjective);
                     pendingGroups.put(key, nextObjective);
+                    groupService.addGroup(groupDescription);
                 }
                 }
                 break;
                 break;
             case HASHED:
             case HASHED:
-                log.debug("processing HASHED next objective");
+                // we convert MPLS ECMP groups to flow-actions for a single
+                // bucket(output port).
+                boolean mplsEcmp = false;
+                if (nextObjective.meta() != null) {
+                    for (Criterion c : nextObjective.meta().criteria()) {
+                        if (c.type() == Type.MPLS_LABEL) {
+                            mplsEcmp = true;
+                        }
+                    }
+                }
+                if (mplsEcmp) {
+                    // covert to flow-actions in a dummy group by choosing the first bucket
+                    log.debug("Converting HASHED group for next objective id {} " +
+                              "to flow-actions in device:{}", nextObjective.id(),
+                              deviceId);
+                    TrafficTreatment treatment = nextObjective.next().iterator().next();
+                    flowObjectiveStore.putNextGroup(nextObjective.id(),
+                                                    new SpringOpenGroup(null, treatment));
+                } else {
+                    // process as ECMP group
                     buckets = nextObjective
                     buckets = nextObjective
                             .next()
                             .next()
                             .stream()
                             .stream()
@@ -289,8 +336,7 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                             .collect(Collectors.toList());
                             .collect(Collectors.toList());
                     if (!buckets.isEmpty()) {
                     if (!buckets.isEmpty()) {
                         final GroupKey key = new DefaultGroupKey(
                         final GroupKey key = new DefaultGroupKey(
-                            appKryo.serialize(nextObjective
-                                    .id()));
+                                                     appKryo.serialize(nextObjective.id()));
                         GroupDescription groupDescription = new DefaultGroupDescription(
                         GroupDescription groupDescription = new DefaultGroupDescription(
                                                   deviceId,
                                                   deviceId,
                                                   GroupDescription.Type.SELECT,
                                                   GroupDescription.Type.SELECT,
@@ -298,14 +344,14 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                                                   key,
                                                   key,
                                                   null,
                                                   null,
                                                   nextObjective.appId());
                                                   nextObjective.appId());
-                    log.debug("Creating HASHED group for next objective id {}",
-                              nextObjective.id());
-                    groupService.addGroup(groupDescription);
+                        log.debug("Creating HASHED group for next objective id {}"
+                                + " in dev:{}", nextObjective.id(), deviceId);
                         pendingGroups.put(key, nextObjective);
                         pendingGroups.put(key, nextObjective);
+                        groupService.addGroup(groupDescription);
+                    }
                 }
                 }
                 break;
                 break;
             case BROADCAST:
             case BROADCAST:
-                log.debug("processing BROADCAST next objective");
                 buckets = nextObjective
                 buckets = nextObjective
                         .next()
                         .next()
                         .stream()
                         .stream()
@@ -323,10 +369,10 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                             key,
                             key,
                             null,
                             null,
                             nextObjective.appId());
                             nextObjective.appId());
-                    log.debug("Creating BROADCAST group for next objective id {}",
-                              nextObjective.id());
-                    groupService.addGroup(groupDescription);
+                    log.debug("Creating BROADCAST group for next objective id {} "
+                            + "in device {}", nextObjective.id(), deviceId);
                     pendingGroups.put(key, nextObjective);
                     pendingGroups.put(key, nextObjective);
+                    groupService.addGroup(groupDescription);
                 }
                 }
                 break;
                 break;
             case FAILOVER:
             case FAILOVER:
@@ -417,9 +463,8 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
     }
     }
 
 
     private Collection<FlowRule> processVersatile(ForwardingObjective fwd) {
     private Collection<FlowRule> processVersatile(ForwardingObjective fwd) {
-        log.debug("Processing versatile forwarding objective");
+        log.debug("Processing versatile forwarding objective in dev:{}", deviceId);
         TrafficSelector selector = fwd.selector();
         TrafficSelector selector = fwd.selector();
-        TrafficTreatment treatment = null;
         EthTypeCriterion ethType =
         EthTypeCriterion ethType =
                 (EthTypeCriterion) selector.getCriterion(Criterion.Type.ETH_TYPE);
                 (EthTypeCriterion) selector.getCriterion(Criterion.Type.ETH_TYPE);
         if (ethType == null) {
         if (ethType == null) {
@@ -428,50 +473,60 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
             return Collections.emptySet();
             return Collections.emptySet();
         }
         }
 
 
+        if (fwd.treatment() == null && fwd.nextId() == null) {
+            log.error("VERSATILE forwarding objective needs next objective ID "
+                    + "or treatment.");
+            return Collections.emptySet();
+        }
+        // emulation of ACL table (for versatile fwd objective) requires
+        // overriding any previous instructions
         TrafficTreatment.Builder treatmentBuilder = DefaultTrafficTreatment
         TrafficTreatment.Builder treatmentBuilder = DefaultTrafficTreatment
                 .builder();
                 .builder();
         treatmentBuilder.wipeDeferred();
         treatmentBuilder.wipeDeferred();
 
 
         if (fwd.nextId() != null) {
         if (fwd.nextId() != null) {
             NextGroup next = flowObjectiveStore.getNextGroup(fwd.nextId());
             NextGroup next = flowObjectiveStore.getNextGroup(fwd.nextId());
-
             if (next != null) {
             if (next != null) {
-                GroupKey key = appKryo.deserialize(next.data());
-
+                SpringOpenGroup soGroup = appKryo.deserialize(next.data());
+                if (soGroup.dummy) {
+                    // need to convert to flow-actions
+                    for (Instruction ins : soGroup.treatment.allInstructions()) {
+                        treatmentBuilder.add(ins);
+                    }
+                } else {
+                    GroupKey key = soGroup.key;
                     Group group = groupService.getGroup(deviceId, key);
                     Group group = groupService.getGroup(deviceId, key);
-
                     if (group == null) {
                     if (group == null) {
                         log.warn("The group left!");
                         log.warn("The group left!");
                         fail(fwd, ObjectiveError.GROUPMISSING);
                         fail(fwd, ObjectiveError.GROUPMISSING);
                         return Collections.emptySet();
                         return Collections.emptySet();
                     }
                     }
                     treatmentBuilder.deferred().group(group.id());
                     treatmentBuilder.deferred().group(group.id());
-                treatment = treatmentBuilder.build();
                     log.debug("Adding OUTGROUP action");
                     log.debug("Adding OUTGROUP action");
                 }
                 }
-        } else if (fwd.treatment() != null) {
+            }
+        }
+
+        if (fwd.treatment() != null) {
             if (fwd.treatment().allInstructions().size() == 1 &&
             if (fwd.treatment().allInstructions().size() == 1 &&
                     fwd.treatment().allInstructions().get(0).type() == Instruction.Type.OUTPUT) {
                     fwd.treatment().allInstructions().get(0).type() == Instruction.Type.OUTPUT) {
                 OutputInstruction o = (OutputInstruction) fwd.treatment().allInstructions().get(0);
                 OutputInstruction o = (OutputInstruction) fwd.treatment().allInstructions().get(0);
                 if (o.port() == PortNumber.CONTROLLER) {
                 if (o.port() == PortNumber.CONTROLLER) {
                     treatmentBuilder.punt();
                     treatmentBuilder.punt();
-                    treatment = treatmentBuilder.build();
                 } else {
                 } else {
-                    treatment = fwd.treatment();
+                    treatmentBuilder.add(o);
                 }
                 }
             } else {
             } else {
-                treatment = fwd.treatment();
+                for (Instruction ins : fwd.treatment().allInstructions()) {
+                    treatmentBuilder.add(ins);
+                }
             }
             }
-        } else {
-            log.warn("VERSATILE forwarding objective needs next objective ID "
-                    + "or treatment.");
-            return Collections.emptySet();
         }
         }
 
 
         FlowRule.Builder ruleBuilder = DefaultFlowRule.builder()
         FlowRule.Builder ruleBuilder = DefaultFlowRule.builder()
                 .fromApp(fwd.appId()).withPriority(fwd.priority())
                 .fromApp(fwd.appId()).withPriority(fwd.priority())
                 .forDevice(deviceId).withSelector(fwd.selector())
                 .forDevice(deviceId).withSelector(fwd.selector())
-                .withTreatment(treatment);
+                .withTreatment(treatmentBuilder.build());
 
 
         if (fwd.permanent()) {
         if (fwd.permanent()) {
             ruleBuilder.makePermanent();
             ruleBuilder.makePermanent();
@@ -508,7 +563,8 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
     }
     }
 
 
     protected Collection<FlowRule> processSpecific(ForwardingObjective fwd) {
     protected Collection<FlowRule> processSpecific(ForwardingObjective fwd) {
-        log.debug("Processing specific");
+        log.debug("Processing specific fwd objective:{} in dev:{} with next:{}",
+                  fwd.id(), deviceId, fwd.nextId());
         boolean isEthTypeObj = isSupportedEthTypeObjective(fwd);
         boolean isEthTypeObj = isSupportedEthTypeObjective(fwd);
         boolean isEthDstObj = isSupportedEthDstObjective(fwd);
         boolean isEthDstObj = isSupportedEthDstObjective(fwd);
 
 
@@ -518,7 +574,7 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
             return processEthDstSpecificObjective(fwd);
             return processEthDstSpecificObjective(fwd);
         } else {
         } else {
             log.warn("processSpecific: Unsupported "
             log.warn("processSpecific: Unsupported "
-                    + "forwarding objective criteraia");
+                    + "forwarding objective criteria");
             fail(fwd, ObjectiveError.UNSUPPORTED);
             fail(fwd, ObjectiveError.UNSUPPORTED);
             return Collections.emptySet();
             return Collections.emptySet();
         }
         }
@@ -540,7 +596,8 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                         .getCriterion(Criterion.Type.IPV4_DST))
                         .getCriterion(Criterion.Type.IPV4_DST))
                         .ip());
                         .ip());
             forTableId = ipv4UnicastTableId;
             forTableId = ipv4UnicastTableId;
-            log.debug("processing IPv4 specific forwarding objective");
+            log.debug("processing IPv4 specific forwarding objective:{} in dev:{}",
+                      fwd.id(), deviceId);
         } else {
         } else {
             filteredSelectorBuilder = filteredSelectorBuilder
             filteredSelectorBuilder = filteredSelectorBuilder
                 .matchEthType(Ethernet.MPLS_UNICAST)
                 .matchEthType(Ethernet.MPLS_UNICAST)
@@ -550,7 +607,8 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
             //if (selector.getCriterion(Criterion.Type.MPLS_BOS) != null) {
             //if (selector.getCriterion(Criterion.Type.MPLS_BOS) != null) {
             //}
             //}
             forTableId = mplsTableId;
             forTableId = mplsTableId;
-            log.debug("processing MPLS specific forwarding objective");
+            log.debug("processing MPLS specific forwarding objective:{} in dev:{}",
+                    fwd.id(), deviceId);
         }
         }
 
 
         TrafficTreatment.Builder treatmentBuilder = DefaultTrafficTreatment
         TrafficTreatment.Builder treatmentBuilder = DefaultTrafficTreatment
@@ -561,24 +619,28 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
             }
             }
         }
         }
 
 
-        //TODO: Analyze the forwarding objective here to make
-        //device specific decision such as no ECMP groups in Dell
-        //switches.
         if (fwd.nextId() != null) {
         if (fwd.nextId() != null) {
             NextGroup next = flowObjectiveStore.getNextGroup(fwd.nextId());
             NextGroup next = flowObjectiveStore.getNextGroup(fwd.nextId());
-
             if (next != null) {
             if (next != null) {
-                GroupKey key = appKryo.deserialize(next.data());
-
+                SpringOpenGroup soGroup = appKryo.deserialize(next.data());
+                if (soGroup.dummy) {
+                    log.debug("Adding flow-actions for fwd. obj. {} "
+                            + "in dev: {}", fwd.id(), deviceId);
+                    for (Instruction ins : soGroup.treatment.allInstructions()) {
+                        treatmentBuilder.add(ins);
+                    }
+                } else {
+                    GroupKey key = soGroup.key;
                     Group group = groupService.getGroup(deviceId, key);
                     Group group = groupService.getGroup(deviceId, key);
-
                     if (group == null) {
                     if (group == null) {
                         log.warn("The group left!");
                         log.warn("The group left!");
                         fail(fwd, ObjectiveError.GROUPMISSING);
                         fail(fwd, ObjectiveError.GROUPMISSING);
                         return Collections.emptySet();
                         return Collections.emptySet();
                     }
                     }
                     treatmentBuilder.deferred().group(group.id());
                     treatmentBuilder.deferred().group(group.id());
-                log.debug("Adding OUTGROUP action");
+                    log.debug("Adding OUTGROUP action to group:{} for fwd. obj. {} "
+                            + "in dev: {}", group.id(), fwd.id(), deviceId);
+                }
             } else {
             } else {
                 log.warn("processSpecific: No associated next objective object");
                 log.warn("processSpecific: No associated next objective object");
                 fail(fwd, ObjectiveError.GROUPMISSING);
                 fail(fwd, ObjectiveError.GROUPMISSING);
@@ -621,6 +683,12 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
         // Do not match MacAddress for subnet broadcast entry
         // Do not match MacAddress for subnet broadcast entry
         if (!ethCriterion.mac().equals(MacAddress.NONE)) {
         if (!ethCriterion.mac().equals(MacAddress.NONE)) {
             filteredSelectorBuilder.matchEthDst(ethCriterion.mac());
             filteredSelectorBuilder.matchEthDst(ethCriterion.mac());
+            log.debug("processing L2 forwarding objective:{} in dev:{}",
+                      fwd.id(), deviceId);
+        } else {
+            log.debug("processing L2 Broadcast forwarding objective:{} "
+                    + "in dev:{} for vlan:{}",
+                      fwd.id(), deviceId, vlanIdCriterion.vlanId());
         }
         }
         filteredSelectorBuilder.matchVlanId(vlanIdCriterion.vlanId());
         filteredSelectorBuilder.matchVlanId(vlanIdCriterion.vlanId());
         TrafficSelector filteredSelector = filteredSelectorBuilder.build();
         TrafficSelector filteredSelector = filteredSelectorBuilder.build();
@@ -635,15 +703,25 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
         if (fwd.nextId() != null) {
         if (fwd.nextId() != null) {
             NextGroup next = flowObjectiveStore.getNextGroup(fwd.nextId());
             NextGroup next = flowObjectiveStore.getNextGroup(fwd.nextId());
             if (next != null) {
             if (next != null) {
-                GroupKey key = appKryo.deserialize(next.data());
-                Group group = groupService.getGroup(deviceId, key);
-                if (group != null) {
-                    treatmentBuilder.deferred().group(group.id());
+                SpringOpenGroup soGrp = appKryo.deserialize(next.data());
+                if (soGrp.dummy) {
+                    log.debug("Adding flow-actions for fwd. obj. {} "
+                            + "in dev: {}", fwd.id(), deviceId);
+                    for (Instruction ins : soGrp.treatment.allInstructions()) {
+                        treatmentBuilder.add(ins);
+                    }
                 } else {
                 } else {
-                    log.warn("Group Missing");
+                    GroupKey key = soGrp.key;
+                    Group group = groupService.getGroup(deviceId, key);
+                    if (group == null) {
+                        log.warn("The group left!");
                         fail(fwd, ObjectiveError.GROUPMISSING);
                         fail(fwd, ObjectiveError.GROUPMISSING);
                         return Collections.emptySet();
                         return Collections.emptySet();
                     }
                     }
+                    treatmentBuilder.deferred().group(group.id());
+                    log.debug("Adding OUTGROUP action to group:{} for fwd. obj. {} "
+                            + "in dev: {}", group.id(), fwd.id(), deviceId);
+                }
             }
             }
         }
         }
         treatmentBuilder.immediate().transition(aclTableId);
         treatmentBuilder.immediate().transition(aclTableId);
@@ -869,14 +947,14 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
             public void onSuccess(FlowRuleOperations ops) {
             public void onSuccess(FlowRuleOperations ops) {
                 pass(filt);
                 pass(filt);
                 log.debug("Provisioned tables in {} with fitering "
                 log.debug("Provisioned tables in {} with fitering "
-                        + "rules for segment router", deviceId);
+                        + "rules", deviceId);
             }
             }
 
 
             @Override
             @Override
             public void onError(FlowRuleOperations ops) {
             public void onError(FlowRuleOperations ops) {
                 fail(filt, ObjectiveError.FLOWINSTALLATIONFAILED);
                 fail(filt, ObjectiveError.FLOWINSTALLATIONFAILED);
                 log.warn("Failed to provision tables in {} with "
                 log.warn("Failed to provision tables in {} with "
-                        + "fitering rules for segment router", deviceId);
+                        + "fitering rules", deviceId);
             }
             }
         }));
         }));
     }
     }
@@ -934,15 +1012,17 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
         @Override
         @Override
         public void event(GroupEvent event) {
         public void event(GroupEvent event) {
             if (event.type() == GroupEvent.Type.GROUP_ADDED) {
             if (event.type() == GroupEvent.Type.GROUP_ADDED) {
-                log.debug("InnerGroupListener: Group ADDED "
+                log.trace("InnerGroupListener: Group ADDED "
                         + "event received in device {}", deviceId);
                         + "event received in device {}", deviceId);
                 GroupKey key = event.subject().appCookie();
                 GroupKey key = event.subject().appCookie();
 
 
                 NextObjective obj = pendingGroups.getIfPresent(key);
                 NextObjective obj = pendingGroups.getIfPresent(key);
                 if (obj != null) {
                 if (obj != null) {
+                    log.debug("Group verified: dev:{} gid:{} <<->> nextId:{}",
+                              deviceId, event.subject().id(), obj.id());
                     flowObjectiveStore
                     flowObjectiveStore
                             .putNextGroup(obj.id(),
                             .putNextGroup(obj.id(),
-                                          new SegmentRoutingGroup(key));
+                                          new SpringOpenGroup(key, null));
                     pass(obj);
                     pass(obj);
                     pendingGroups.invalidate(key);
                     pendingGroups.invalidate(key);
                 }
                 }
@@ -971,21 +1051,47 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                                  if (obj == null) {
                                  if (obj == null) {
                                      return;
                                      return;
                                  }
                                  }
+                                 log.debug("Group verified: dev:{} gid:{} <<->> nextId:{}",
+                                           deviceId,
+                                           groupService.getGroup(deviceId, key).id(),
+                                           obj.id());
                                  pass(obj);
                                  pass(obj);
                                  pendingGroups.invalidate(key);
                                  pendingGroups.invalidate(key);
-                                 flowObjectiveStore.putNextGroup(obj.id(),
-                                                                 new SegmentRoutingGroup(
-                                                                                         key));
+                                 flowObjectiveStore.putNextGroup(
+                                                        obj.id(),
+                                                        new SpringOpenGroup(key, null));
                     });
                     });
         }
         }
     }
     }
 
 
-    private class SegmentRoutingGroup implements NextGroup {
-
+    /**
+     * SpringOpenGroup can either serve as storage for a GroupKey which can be
+     * used to fetch the group from the Group Service, or it can be serve as storage
+     * for Traffic Treatments which can be used as flow actions. In the latter
+     * case, we refer to this as a dummy group.
+     *
+     */
+    private class SpringOpenGroup implements NextGroup {
+        private final boolean dummy;
         private final GroupKey key;
         private final GroupKey key;
+        private final TrafficTreatment treatment;
 
 
-        public SegmentRoutingGroup(GroupKey key) {
+        /**
+         * Storage for a GroupKey or a TrafficTreatment. One of the params
+         * to this constructor must be null.
+         * @param key represents a GroupKey
+         * @param treatment represents flow actions in a dummy group
+         */
+        public SpringOpenGroup(GroupKey key, TrafficTreatment treatment) {
+            if (key == null) {
+                this.key = new DefaultGroupKey(new byte[]{0});
+                this.treatment = treatment;
+                this.dummy = true;
+            } else {
                 this.key = key;
                 this.key = key;
+                this.treatment = DefaultTrafficTreatment.builder().build();
+                this.dummy = false;
+            }
         }
         }
 
 
         @SuppressWarnings("unused")
         @SuppressWarnings("unused")
@@ -995,7 +1101,7 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
 
 
         @Override
         @Override
         public byte[] data() {
         public byte[] data() {
-            return appKryo.serialize(key);
+            return appKryo.serialize(this);
         }
         }
 
 
     }
     }