CORD-367 L2 bridging and L3 routing support with internal VLANs in OF-DPA.

Also includes:
     All forwarding in app is now via nextObjectives (not treatments) - Spring Open driver converts
     non-ECMP forwarding to flow-actions, while OF-DPA driver continues to use groups.
     Convert 'setMeta' methods to 'withMeta' in Flow Objectives API.
     Bug fix in Flow Objective Manager - set of PendingNext is now threadsafe.
     Bug fix in ArpHandler - now recognizes routerIp in addition to gatewayIps
     Removed a bunch of testcode
     Added group count in CLI

Change-Id: Id3b879c5dda78151ca0ec359179f1604066d39fc

@@ -107,7 +107,7 @@ public class ArpHandler {
                                             vlanId);
                                             vlanId);
 
 
         // ARP request for router. Send ARP reply.
         // ARP request for router. Send ARP reply.
-        if (isArpReqForRouter(deviceId, arpRequest)) {
+        if (isArpForRouter(deviceId, arpRequest)) {
             Ip4Address targetAddress = Ip4Address.valueOf(arpRequest.getTargetProtocolAddress());
             Ip4Address targetAddress = Ip4Address.valueOf(arpRequest.getTargetProtocolAddress());
             sendArpResponse(arpRequest, config.getRouterMacForAGatewayIp(targetAddress), vlanId);
             sendArpResponse(arpRequest, config.getRouterMacForAGatewayIp(targetAddress), vlanId);
         } else {
         } else {
@@ -130,7 +130,7 @@ public class ArpHandler {
                                             vlanId);
                                             vlanId);
 
 
         // ARP reply for router. Process all pending IP packets.
         // ARP reply for router. Process all pending IP packets.
-        if (isArpReqForRouter(deviceId, arpReply)) {
+        if (isArpForRouter(deviceId, arpReply)) {
             Ip4Address hostIpAddress = Ip4Address.valueOf(arpReply.getSenderProtocolAddress());
             Ip4Address hostIpAddress = Ip4Address.valueOf(arpReply.getSenderProtocolAddress());
             srManager.ipHandler.forwardPackets(deviceId, hostIpAddress);
             srManager.ipHandler.forwardPackets(deviceId, hostIpAddress);
         } else {
         } else {
@@ -141,7 +141,8 @@ public class ArpHandler {
             // ARP reply for unknown host, Flood in the subnet.
             // ARP reply for unknown host, Flood in the subnet.
             } else {
             } else {
                 // Don't flood to non-edge ports
                 // Don't flood to non-edge ports
-                if (vlanId.equals(VlanId.vlanId(srManager.ASSIGNED_VLAN_NO_SUBNET))) {
+                if (vlanId.equals(
+                        VlanId.vlanId(SegmentRoutingManager.ASSIGNED_VLAN_NO_SUBNET))) {
                     return;
                     return;
                 }
                 }
                 removeVlanAndFlood(payload, inPort);
                 removeVlanAndFlood(payload, inPort);
@@ -150,14 +151,21 @@ public class ArpHandler {
     }
     }
 
 
 
 
-    private boolean isArpReqForRouter(DeviceId deviceId, ARP arpRequest) {
-        Set<Ip4Address> gatewayIpAddresses = config.getPortIPs(deviceId);
-        if (gatewayIpAddresses != null) {
-            Ip4Address targetProtocolAddress = Ip4Address.valueOf(arpRequest
-                    .getTargetProtocolAddress());
-            if (gatewayIpAddresses.contains(targetProtocolAddress)) {
+    private boolean isArpForRouter(DeviceId deviceId, ARP arpMsg) {
+        Ip4Address targetProtocolAddress = Ip4Address.valueOf(
+                                               arpMsg.getTargetProtocolAddress());
+        Set<Ip4Address> gatewayIpAddresses = null;
+        try {
+            if (targetProtocolAddress.equals(config.getRouterIp(deviceId))) {
                 return true;
                 return true;
             }
             }
+            gatewayIpAddresses = config.getPortIPs(deviceId);
+        } catch (DeviceConfigNotFoundException e) {
+            log.warn(e.getMessage() + " Aborting check for router IP in processing arp");
+        }
+        if (gatewayIpAddresses != null &&
+                gatewayIpAddresses.contains(targetProtocolAddress)) {
+            return true;
         }
         }
         return false;
         return false;
     }
     }

@@ -88,10 +88,10 @@ public class IcmpHandler {
                 (destinationAddress.equals(routerIpAddress) ||
                 (destinationAddress.equals(routerIpAddress) ||
                         gatewayIpAddresses.contains(destinationAddress))) {
                         gatewayIpAddresses.contains(destinationAddress))) {
             sendICMPResponse(ethernet, connectPoint);
             sendICMPResponse(ethernet, connectPoint);
-            // TODO: do we need to set the flow rule again ??
 
 
         // ICMP for any known host
         // ICMP for any known host
         } else if (!srManager.hostService.getHostsByIp(destinationAddress).isEmpty()) {
         } else if (!srManager.hostService.getHostsByIp(destinationAddress).isEmpty()) {
+            // TODO: known host packet should not be coming to controller - resend flows?
             srManager.ipHandler.forwardPackets(deviceId, destinationAddress);
             srManager.ipHandler.forwardPackets(deviceId, destinationAddress);
 
 
         // ICMP for an unknown host in the subnet of the router
         // ICMP for an unknown host in the subnet of the router

@@ -98,7 +98,7 @@ public class IpHandler {
      */
      */
     public void addToPacketBuffer(IPv4 ipPacket) {
     public void addToPacketBuffer(IPv4 ipPacket) {
 
 
-        // Better not buffer TPC packets due to out-of-order packet transfer
+        // Better not buffer TCP packets due to out-of-order packet transfer
         if (ipPacket.getProtocol() == IPv4.PROTOCOL_TCP) {
         if (ipPacket.getProtocol() == IPv4.PROTOCOL_TCP) {
             return;
             return;
         }
         }

@@ -147,20 +147,34 @@ public class RoutingRulePopulator {
         TrafficSelector.Builder sbuilder = DefaultTrafficSelector.builder();
         TrafficSelector.Builder sbuilder = DefaultTrafficSelector.builder();
         TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
         TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
 
 
-        sbuilder.matchIPDst(IpPrefix.valueOf(hostIp, IpPrefix.MAX_INET_MASK_LENGTH));
         sbuilder.matchEthType(Ethernet.TYPE_IPV4);
         sbuilder.matchEthType(Ethernet.TYPE_IPV4);
+        sbuilder.matchIPDst(IpPrefix.valueOf(hostIp, IpPrefix.MAX_INET_MASK_LENGTH));
+        TrafficSelector selector = sbuilder.build();
 
 
         tbuilder.deferred()
         tbuilder.deferred()
                 .setEthDst(hostMac)
                 .setEthDst(hostMac)
                 .setEthSrc(deviceMac)
                 .setEthSrc(deviceMac)
                 .setOutput(outPort);
                 .setOutput(outPort);
-
         TrafficTreatment treatment = tbuilder.build();
         TrafficTreatment treatment = tbuilder.build();
-        TrafficSelector selector = sbuilder.build();
+
+        // All forwarding is via Groups. Drivers can re-purpose to flow-actions if needed.
+        // for switch pipelines that need it, provide outgoing vlan as metadata
+        VlanId outvlan = null;
+        Ip4Prefix subnet = srManager.deviceConfiguration.getPortSubnet(deviceId, outPort);
+        if (subnet == null) {
+            outvlan = VlanId.vlanId(SegmentRoutingManager.ASSIGNED_VLAN_NO_SUBNET);
+        } else {
+            outvlan = srManager.getSubnetAssignedVlanId(deviceId, subnet);
+        }
+        TrafficSelector meta = DefaultTrafficSelector.builder()
+                                    .matchVlanId(outvlan).build();
+        int portNextObjId = srManager.getPortNextObjectiveId(deviceId, outPort,
+                                                             treatment, meta);
 
 
         return DefaultForwardingObjective.builder()
         return DefaultForwardingObjective.builder()
+                .withSelector(selector)
+                .nextStep(portNextObjId)
                 .fromApp(srManager.appId).makePermanent()
                 .fromApp(srManager.appId).makePermanent()
-                .withSelector(selector).withTreatment(treatment)
                 .withPriority(100).withFlag(ForwardingObjective.Flag.SPECIFIC);
                 .withPriority(100).withFlag(ForwardingObjective.Flag.SPECIFIC);
     }
     }
 
 
@@ -454,7 +468,7 @@ public class RoutingRulePopulator {
                 if (srManager.mastershipService.isLocalMaster(deviceId)) {
                 if (srManager.mastershipService.isLocalMaster(deviceId)) {
                     TrafficTreatment tt = DefaultTrafficTreatment.builder()
                     TrafficTreatment tt = DefaultTrafficTreatment.builder()
                             .pushVlan().setVlanId(assignedVlan).build();
                             .pushVlan().setVlanId(assignedVlan).build();
-                    fob.setMeta(tt);
+                    fob.withMeta(tt);
                 }
                 }
                 fob.permit().fromApp(srManager.appId);
                 fob.permit().fromApp(srManager.appId);
                 srManager.flowObjectiveService.
                 srManager.flowObjectiveService.
@@ -559,6 +573,12 @@ public class RoutingRulePopulator {
             int nextId = srManager.getSubnetNextObjectiveId(deviceId, subnet);
             int nextId = srManager.getSubnetNextObjectiveId(deviceId, subnet);
             VlanId vlanId = srManager.getSubnetAssignedVlanId(deviceId, subnet);
             VlanId vlanId = srManager.getSubnetAssignedVlanId(deviceId, subnet);
 
 
+            if (nextId < 0 || vlanId == null) {
+                log.error("Cannot install subnet broadcast rule in dev:{} due"
+                        + "to vlanId:{} or nextId:{}", vlanId, nextId);
+                return;
+            }
+
             /* Driver should treat objective with MacAddress.NONE as the
             /* Driver should treat objective with MacAddress.NONE as the
              * subnet broadcast rule
              * subnet broadcast rule
              */
              */

@@ -56,9 +56,11 @@ public class DefaultEdgeGroupHandler extends DefaultGroupHandler {
                                           NeighborSetNextObjectiveStoreKey,
                                           NeighborSetNextObjectiveStoreKey,
                                           Integer> nsNextObjStore,
                                           Integer> nsNextObjStore,
                                   EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
                                   EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
-                                          Integer> subnetNextObjStore) {
+                                          Integer> subnetNextObjStore,
+                                  EventuallyConsistentMap<PortNextObjectiveStoreKey,
+                                          Integer> portNextObjStore) {
         super(deviceId, appId, config, linkService, flowObjService,
         super(deviceId, appId, config, linkService, flowObjService,
-              nsNextObjStore, subnetNextObjStore);
+              nsNextObjStore, subnetNextObjStore, portNextObjStore);
     }
     }
 
 
     @Override
     @Override

@@ -50,9 +50,11 @@ public class DefaultTransitGroupHandler extends DefaultGroupHandler {
                                         NeighborSetNextObjectiveStoreKey,
                                         NeighborSetNextObjectiveStoreKey,
                                         Integer> nsNextObjStore,
                                         Integer> nsNextObjStore,
                                   EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
                                   EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
-                                        Integer> subnetNextObjStore) {
+                                        Integer> subnetNextObjStore,
+                                  EventuallyConsistentMap<PortNextObjectiveStoreKey,
+                                  Integer> portNextObjStore) {
         super(deviceId, appId, config, linkService, flowObjService,
         super(deviceId, appId, config, linkService, flowObjService,
-              nsNextObjStore, subnetNextObjStore);
+              nsNextObjStore, subnetNextObjStore, portNextObjStore);
     }
     }
 
 
     @Override
     @Override

@@ -68,9 +68,11 @@ public class PolicyGroupHandler extends DefaultGroupHandler {
                               EventuallyConsistentMap<NeighborSetNextObjectiveStoreKey,
                               EventuallyConsistentMap<NeighborSetNextObjectiveStoreKey,
                                       Integer> nsNextObjStore,
                                       Integer> nsNextObjStore,
                               EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
                               EventuallyConsistentMap<SubnetNextObjectiveStoreKey,
-                                      Integer> subnetNextObjStore) {
+                                      Integer> subnetNextObjStore,
+                              EventuallyConsistentMap<PortNextObjectiveStoreKey,
+                              Integer> portNextObjStore) {
         super(deviceId, appId, config, linkService, flowObjService,
         super(deviceId, appId, config, linkService, flowObjService,
-              nsNextObjStore, subnetNextObjStore);
+              nsNextObjStore, subnetNextObjStore, portNextObjStore);
     }
     }
 
 
     public PolicyGroupIdentifier createPolicyGroupChain(String id,
     public PolicyGroupIdentifier createPolicyGroupChain(String id,

+package org.onosproject.segmentrouting.grouphandler;
+
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.flow.TrafficTreatment;
+
+import java.util.Objects;
+
+/**
+ * Class definition of Key for Device/Port to NextObjective store. Since there
+ * can be multiple next objectives to the same physical port, we differentiate
+ * between them by including the treatment in the key.
+ */
+public class PortNextObjectiveStoreKey {
+    private final DeviceId deviceId;
+    private final PortNumber portNum;
+    private final TrafficTreatment treatment;
+
+    public PortNextObjectiveStoreKey(DeviceId deviceId, PortNumber portNum,
+                                     TrafficTreatment treatment) {
+        this.deviceId = deviceId;
+        this.portNum = portNum;
+        this.treatment = treatment;
+    }
+
+    /**
+     * Gets device id in this PortNextObjectiveStoreKey.
+     *
+     * @return device id
+     */
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    /**
+     * Gets port information in this PortNextObjectiveStoreKey.
+     *
+     * @return port information
+     */
+    public PortNumber portNumber() {
+        return portNum;
+    }
+
+    /**
+     * Gets treatment information in this PortNextObjectiveStoreKey.
+     *
+     * @return treatment information
+     */
+    public TrafficTreatment treatment() {
+        return treatment;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof PortNextObjectiveStoreKey)) {
+            return false;
+        }
+        PortNextObjectiveStoreKey that =
+                (PortNextObjectiveStoreKey) o;
+        return (Objects.equals(this.deviceId, that.deviceId) &&
+                Objects.equals(this.portNum, that.portNum) &&
+                Objects.equals(this.treatment, that.treatment));
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(deviceId, portNum, treatment);
+    }
+
+    @Override
+    public String toString() {
+        return "Device: " + deviceId + " Port: " + portNum + " Treatment: " + treatment;
+    }
+}

@@ -119,7 +119,7 @@ public class GroupsListCommand extends AbstractShellCommand {
     }
     }
 
 
     private void printGroups(DeviceId deviceId, List<Group> groups) {
     private void printGroups(DeviceId deviceId, List<Group> groups) {
-        print("deviceId=%s", deviceId);
+        print("deviceId=%s, groupCount=%s", deviceId, groups.size());
         for (Group group : groups) {
         for (Group group : groups) {
             print(FORMAT, Integer.toHexString(group.id().id()), group.state(), group.type(),
             print(FORMAT, Integer.toHexString(group.id().id()), group.state(), group.type(),
                   group.bytes(), group.packets(), group.appId().name());
                   group.bytes(), group.packets(), group.appId().name());

@@ -196,7 +196,7 @@ public final class DefaultFilteringObjective implements FilteringObjective {
         }
         }
 
 
         @Override
         @Override
-        public Builder setMeta(TrafficTreatment treatment) {
+        public Builder withMeta(TrafficTreatment treatment) {
             this.meta = treatment;
             this.meta = treatment;
             return this;
             return this;
         }
         }

@@ -181,7 +181,7 @@ public final class DefaultNextObjective implements NextObjective {
         }
         }
 
 
         @Override
         @Override
-        public Builder setMeta(TrafficSelector meta) {
+        public Builder withMeta(TrafficSelector meta) {
             this.meta = meta;
             this.meta = meta;
             return this;
             return this;
         }
         }

@@ -133,7 +133,7 @@ public interface FilteringObjective extends Objective {
          * @param treatment traffic treatment to use
          * @param treatment traffic treatment to use
          * @return a filtering builder
          * @return a filtering builder
          */
          */
-        Builder setMeta(TrafficTreatment treatment);
+        Builder withMeta(TrafficTreatment treatment);
 
 
         /**
         /**
          * Assigns an application id.
          * Assigns an application id.

@@ -147,7 +147,7 @@ public interface NextObjective extends Objective {
          * @param selector match conditions
          * @param selector match conditions
          * @return an objective builder
          * @return an objective builder
          */
          */
-        Builder setMeta(TrafficSelector selector);
+        Builder withMeta(TrafficSelector selector);
 
 
         /**
         /**
          * Builds the next objective that will be added.
          * Builds the next objective that will be added.

@@ -16,7 +16,6 @@
 package org.onosproject.net.flowobjective.impl;
 package org.onosproject.net.flowobjective.impl;
 
 
 import com.google.common.collect.Maps;
 import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -53,9 +52,11 @@ import org.onosproject.net.group.GroupService;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;
 
 
+import java.util.Collections;
 import java.util.Map;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Objects;
 import java.util.Set;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ExecutorService;
 
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -228,8 +229,10 @@ public class FlowObjectiveManager implements FlowObjectiveService {
                 flowObjectiveStore.getNextGroup(fwd.nextId()) == null) {
                 flowObjectiveStore.getNextGroup(fwd.nextId()) == null) {
             log.trace("Queuing forwarding objective for nextId {}", fwd.nextId());
             log.trace("Queuing forwarding objective for nextId {}", fwd.nextId());
             // TODO: change to computeIfAbsent
             // TODO: change to computeIfAbsent
-            Set<PendingNext> pnext = pendingForwards.putIfAbsent(fwd.nextId(),
-                                         Sets.newHashSet(new PendingNext(deviceId, fwd)));
+            Set<PendingNext> newset = Collections.newSetFromMap(
+                                          new ConcurrentHashMap<PendingNext, Boolean>());
+            newset.add(new PendingNext(deviceId, fwd));
+            Set<PendingNext> pnext = pendingForwards.putIfAbsent(fwd.nextId(), newset);
             if (pnext != null) {
             if (pnext != null) {
                 pnext.add(new PendingNext(deviceId, fwd));
                 pnext.add(new PendingNext(deviceId, fwd));
             }
             }

@@ -350,7 +350,7 @@ public class DistributedGroupStore
         // Check if a group is existing with the same key
         // Check if a group is existing with the same key
         Group existingGroup = getGroup(groupDesc.deviceId(), groupDesc.appCookie());
         Group existingGroup = getGroup(groupDesc.deviceId(), groupDesc.appCookie());
         if (existingGroup != null) {
         if (existingGroup != null) {
-            log.warn("Group already exists with the same key {} in dev:{} with id:{}",
+            log.warn("Group already exists with the same key {} in dev:{} with id:0x{}",
                      groupDesc.appCookie(), groupDesc.deviceId(),
                      groupDesc.appCookie(), groupDesc.deviceId(),
                      Integer.toHexString(existingGroup.id().id()));
                      Integer.toHexString(existingGroup.id().id()));
             return;
             return;

@@ -39,7 +39,9 @@ import org.onosproject.net.flow.FlowRuleOperations;
 import org.onosproject.net.flow.FlowRuleOperationsContext;
 import org.onosproject.net.flow.FlowRuleOperationsContext;
 import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.criteria.Criteria;
 import org.onosproject.net.flow.criteria.Criterion;
 import org.onosproject.net.flow.criteria.Criterion;
+import org.onosproject.net.flow.criteria.EthCriterion;
 import org.onosproject.net.flow.criteria.EthTypeCriterion;
 import org.onosproject.net.flow.criteria.EthTypeCriterion;
 import org.onosproject.net.flow.criteria.IPCriterion;
 import org.onosproject.net.flow.criteria.IPCriterion;
 import org.onosproject.net.flow.criteria.MplsBosCriterion;
 import org.onosproject.net.flow.criteria.MplsBosCriterion;
@@ -62,6 +64,14 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
 
 
     private final Logger log = getLogger(getClass());
     private final Logger log = getLogger(getClass());
 
 
+    /*
+     * Cpqd emulation does not require the non-OF standard rules for
+     * matching untagged packets.
+     *
+     * (non-Javadoc)
+     * @see org.onosproject.driver.pipeline.OFDPA2Pipeline#processVlanIdFilter
+     */
+
     @Override
     @Override
     protected List<FlowRule> processVlanIdFilter(PortCriterion portCriterion,
     protected List<FlowRule> processVlanIdFilter(PortCriterion portCriterion,
                                                  VlanIdCriterion vidCriterion,
                                                  VlanIdCriterion vidCriterion,
@@ -122,16 +132,101 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
         return rules;
         return rules;
     }
     }
 
 
+    /*
+     * Cpqd emulation does not handle vlan tags and mpls labels correctly.
+     * Workaround requires popping off the VLAN tags in the TMAC table.
+     *
+     * (non-Javadoc)
+     * @see org.onosproject.driver.pipeline.OFDPA2Pipeline#processEthDstFilter
+     */
     @Override
     @Override
-    protected Collection<FlowRule> processSpecific(ForwardingObjective fwd) {
+    protected List<FlowRule> processEthDstFilter(PortCriterion portCriterion,
+                                                 EthCriterion ethCriterion,
+                                                 VlanIdCriterion vidCriterion,
+                                                 VlanId assignedVlan,
+                                                 ApplicationId applicationId) {
+        //handling untagged packets via assigned VLAN
+        if (vidCriterion.vlanId() == VlanId.NONE) {
+            vidCriterion = (VlanIdCriterion) Criteria.matchVlanId(assignedVlan);
+        }
+        // ofdpa cannot match on ALL portnumber, so we need to use separate
+        // rules for each port.
+        List<PortNumber> portnums = new ArrayList<PortNumber>();
+        if (portCriterion.port() == PortNumber.ALL) {
+            for (Port port : deviceService.getPorts(deviceId)) {
+                if (port.number().toLong() > 0 && port.number().toLong() < OFPP_MAX) {
+                    portnums.add(port.number());
+                }
+            }
+        } else {
+            portnums.add(portCriterion.port());
+        }
+
+        List<FlowRule> rules = new ArrayList<FlowRule>();
+        for (PortNumber pnum : portnums) {
+            // for unicast IP packets
+            TrafficSelector.Builder selector = DefaultTrafficSelector.builder();
+            TrafficTreatment.Builder treatment = DefaultTrafficTreatment.builder();
+            selector.matchInPort(pnum);
+            selector.matchVlanId(vidCriterion.vlanId());
+            selector.matchEthType(Ethernet.TYPE_IPV4);
+            selector.matchEthDst(ethCriterion.mac());
+            /*
+             * Note: CpqD switches do not handle MPLS-related operation properly
+             * for a packet with VLAN tag. We pop VLAN here as a workaround.
+             * Side effect: HostService learns redundant hosts with same MAC but
+             * different VLAN. No known side effect on the network reachability.
+             */
+            treatment.popVlan();
+            treatment.transition(UNICAST_ROUTING_TABLE);
+            FlowRule rule = DefaultFlowRule.builder()
+                    .forDevice(deviceId)
+                    .withSelector(selector.build())
+                    .withTreatment(treatment.build())
+                    .withPriority(DEFAULT_PRIORITY)
+                    .fromApp(applicationId)
+                    .makePermanent()
+                    .forTable(TMAC_TABLE).build();
+            rules.add(rule);
+            //for MPLS packets
+            selector = DefaultTrafficSelector.builder();
+            treatment = DefaultTrafficTreatment.builder();
+            selector.matchInPort(pnum);
+            selector.matchVlanId(vidCriterion.vlanId());
+            selector.matchEthType(Ethernet.MPLS_UNICAST);
+            selector.matchEthDst(ethCriterion.mac());
+            // workaround here again
+            treatment.popVlan();
+            treatment.transition(MPLS_TABLE_0);
+            rule = DefaultFlowRule.builder()
+                    .forDevice(deviceId)
+                    .withSelector(selector.build())
+                    .withTreatment(treatment.build())
+                    .withPriority(DEFAULT_PRIORITY)
+                    .fromApp(applicationId)
+                    .makePermanent()
+                    .forTable(TMAC_TABLE).build();
+            rules.add(rule);
+        }
+        return rules;
+    }
+
+    /*
+     * Cpqd emulation allows MPLS ecmp.
+     *
+     * (non-Javadoc)
+     * @see org.onosproject.driver.pipeline.OFDPA2Pipeline#processEthTypeSpecific
+     */
+    @Override
+    protected Collection<FlowRule> processEthTypeSpecific(ForwardingObjective fwd) {
         TrafficSelector selector = fwd.selector();
         TrafficSelector selector = fwd.selector();
         EthTypeCriterion ethType =
         EthTypeCriterion ethType =
                 (EthTypeCriterion) selector.getCriterion(Criterion.Type.ETH_TYPE);
                 (EthTypeCriterion) selector.getCriterion(Criterion.Type.ETH_TYPE);
         if ((ethType == null) ||
         if ((ethType == null) ||
                 (ethType.ethType().toShort() != Ethernet.TYPE_IPV4) &&
                 (ethType.ethType().toShort() != Ethernet.TYPE_IPV4) &&
                 (ethType.ethType().toShort() != Ethernet.MPLS_UNICAST)) {
                 (ethType.ethType().toShort() != Ethernet.MPLS_UNICAST)) {
-            log.warn("processSpecific: Unsupported "
-                    + "forwarding objective criteraia");
+            log.warn("processSpecific: Unsupported forwarding objective criteria"
+                    + "ethType:{} in dev:{}", ethType, deviceId);
             fail(fwd, ObjectiveError.UNSUPPORTED);
             fail(fwd, ObjectiveError.UNSUPPORTED);
             return Collections.emptySet();
             return Collections.emptySet();
         }
         }
@@ -143,8 +238,8 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
                 .matchIPDst(((IPCriterion)
                 .matchIPDst(((IPCriterion)
                         selector.getCriterion(Criterion.Type.IPV4_DST)).ip());
                         selector.getCriterion(Criterion.Type.IPV4_DST)).ip());
             forTableId = UNICAST_ROUTING_TABLE;
             forTableId = UNICAST_ROUTING_TABLE;
-            log.debug("processing IPv4 specific forwarding objective {} hash{} in dev:{}",
-                      fwd.id(), fwd.hashCode(), deviceId);
+            log.debug("processing IPv4 specific forwarding objective {} -> next:{}"
+                    + " in dev:{}", fwd.id(), fwd.nextId(), deviceId);
         } else {
         } else {
             filteredSelector
             filteredSelector
                 .matchEthType(Ethernet.MPLS_UNICAST)
                 .matchEthType(Ethernet.MPLS_UNICAST)
@@ -156,8 +251,8 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
                 filteredSelector.matchMplsBos(bos.mplsBos());
                 filteredSelector.matchMplsBos(bos.mplsBos());
             }
             }
             forTableId = MPLS_TABLE_1;
             forTableId = MPLS_TABLE_1;
-            log.debug("processing MPLS specific forwarding objective {} hash:{} in dev {}",
-                      fwd.id(), fwd.hashCode(), deviceId);
+            log.debug("processing MPLS specific forwarding objective {} -> next:{}"
+                    + " in dev {}", fwd.id(), fwd.nextId(), deviceId);
         }
         }
 
 
         TrafficTreatment.Builder tb = DefaultTrafficTreatment.builder();
         TrafficTreatment.Builder tb = DefaultTrafficTreatment.builder();
@@ -197,7 +292,6 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
         return Collections.singletonList(ruleBuilder.build());
         return Collections.singletonList(ruleBuilder.build());
     }
     }
 
 
-
     @Override
     @Override
     protected void initializePipeline() {
     protected void initializePipeline() {
         processPortTable();
         processPortTable();
@@ -210,7 +304,6 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
         processAclTable();
         processAclTable();
     }
     }
 
 
-    @Override
     protected void processPortTable() {
     protected void processPortTable() {
         FlowRuleOperations.Builder ops = FlowRuleOperations.builder();
         FlowRuleOperations.Builder ops = FlowRuleOperations.builder();
         TrafficSelector.Builder selector = DefaultTrafficSelector.builder();
         TrafficSelector.Builder selector = DefaultTrafficSelector.builder();
@@ -239,7 +332,6 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
         }));
         }));
     }
     }
 
 
-    @Override
     protected void processTmacTable() {
     protected void processTmacTable() {
         //table miss entry
         //table miss entry
         FlowRuleOperations.Builder ops = FlowRuleOperations.builder();
         FlowRuleOperations.Builder ops = FlowRuleOperations.builder();
@@ -270,7 +362,6 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
         }));
         }));
     }
     }
 
 
-    @Override
     protected void processIpTable() {
     protected void processIpTable() {
         //table miss entry
         //table miss entry
         FlowRuleOperations.Builder ops = FlowRuleOperations.builder();
         FlowRuleOperations.Builder ops = FlowRuleOperations.builder();
@@ -278,6 +369,7 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
         TrafficTreatment.Builder treatment = DefaultTrafficTreatment.builder();
         TrafficTreatment.Builder treatment = DefaultTrafficTreatment.builder();
         selector = DefaultTrafficSelector.builder();
         selector = DefaultTrafficSelector.builder();
         treatment = DefaultTrafficTreatment.builder();
         treatment = DefaultTrafficTreatment.builder();
+        treatment.deferred().setOutput(PortNumber.CONTROLLER);
         treatment.transition(ACL_TABLE);
         treatment.transition(ACL_TABLE);
         FlowRule rule = DefaultFlowRule.builder()
         FlowRule rule = DefaultFlowRule.builder()
                 .forDevice(deviceId)
                 .forDevice(deviceId)
@@ -301,7 +393,6 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
         }));
         }));
     }
     }
 
 
-    @Override
     protected void processMplsTable() {
     protected void processMplsTable() {
         //table miss entry
         //table miss entry
         FlowRuleOperations.Builder ops = FlowRuleOperations.builder();
         FlowRuleOperations.Builder ops = FlowRuleOperations.builder();
@@ -374,7 +465,6 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
         }));
         }));
     }
     }
 
 
-    @Override
     protected void processAclTable() {
     protected void processAclTable() {
         //table miss entry - catch all to executed action-set
         //table miss entry - catch all to executed action-set
         FlowRuleOperations.Builder ops = FlowRuleOperations.builder();
         FlowRuleOperations.Builder ops = FlowRuleOperations.builder();

@@ -287,24 +287,14 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
             case SIMPLE:
             case SIMPLE:
                 Collection<TrafficTreatment> treatments = nextObjective.next();
                 Collection<TrafficTreatment> treatments = nextObjective.next();
                 if (treatments.size() == 1) {
                 if (treatments.size() == 1) {
-                    TrafficTreatment treatment = treatments.iterator().next();
-                    GroupBucket bucket = DefaultGroupBucket
-                            .createIndirectGroupBucket(treatment);
-                    final GroupKey key = new DefaultGroupKey(
-                            appKryo.serialize(nextObjective
-                                    .id()));
-                    GroupDescription groupDescription = new DefaultGroupDescription(
-                            deviceId,
-                            GroupDescription.Type.INDIRECT,
-                            new GroupBuckets(
-                                    Collections.singletonList(bucket)),
-                            key,
-                            null,
-                            nextObjective.appId());
-                    log.debug("Creating SIMPLE group for next objective id {} "
-                            + "in dev:{}", nextObjective.id(), deviceId);
-                    pendingGroups.put(key, nextObjective);
-                    groupService.addGroup(groupDescription);
+                    // Spring Open TTP converts simple nextObjective to flow-actions
+                    // in a dummy group
+                    TrafficTreatment treatment = nextObjective.next().iterator().next();
+                    log.debug("Converting SIMPLE group for next objective id {} " +
+                            "to {} flow-actions in device:{}", nextObjective.id(),
+                            treatment.allInstructions().size(), deviceId);
+                    flowObjectiveStore.putNextGroup(nextObjective.id(),
+                                                    new SpringOpenGroup(null, treatment));
                 }
                 }
                 break;
                 break;
             case HASHED:
             case HASHED:
@@ -624,8 +614,9 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
             if (next != null) {
             if (next != null) {
                 SpringOpenGroup soGroup = appKryo.deserialize(next.data());
                 SpringOpenGroup soGroup = appKryo.deserialize(next.data());
                 if (soGroup.dummy) {
                 if (soGroup.dummy) {
-                    log.debug("Adding flow-actions for fwd. obj. {} "
-                            + "in dev: {}", fwd.id(), deviceId);
+                    log.debug("Adding {} flow-actions for fwd. obj. {} -> next:{} "
+                            + "in dev: {}", soGroup.treatment.allInstructions().size(),
+                            fwd.id(), fwd.nextId(), deviceId);
                     for (Instruction ins : soGroup.treatment.allInstructions()) {
                     for (Instruction ins : soGroup.treatment.allInstructions()) {
                         treatmentBuilder.add(ins);
                         treatmentBuilder.add(ins);
                     }
                     }
@@ -639,7 +630,8 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                     }
                     }
                     treatmentBuilder.deferred().group(group.id());
                     treatmentBuilder.deferred().group(group.id());
                     log.debug("Adding OUTGROUP action to group:{} for fwd. obj. {} "
                     log.debug("Adding OUTGROUP action to group:{} for fwd. obj. {} "
-                            + "in dev: {}", group.id(), fwd.id(), deviceId);
+                            + "for next:{} in dev: {}", group.id(), fwd.id(),
+                            fwd.nextId(), deviceId);
                 }
                 }
             } else {
             } else {
                 log.warn("processSpecific: No associated next objective object");
                 log.warn("processSpecific: No associated next objective object");
@@ -705,10 +697,11 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
             if (next != null) {
             if (next != null) {
                 SpringOpenGroup soGrp = appKryo.deserialize(next.data());
                 SpringOpenGroup soGrp = appKryo.deserialize(next.data());
                 if (soGrp.dummy) {
                 if (soGrp.dummy) {
-                    log.debug("Adding flow-actions for fwd. obj. {} "
-                            + "in dev: {}", fwd.id(), deviceId);
+                    log.debug("Adding {} flow-actions for fwd. obj. {} "
+                            + "in dev: {}", soGrp.treatment.allInstructions().size(),
+                            fwd.id(), deviceId);
                     for (Instruction ins : soGrp.treatment.allInstructions()) {
                     for (Instruction ins : soGrp.treatment.allInstructions()) {
-                        treatmentBuilder.add(ins);
+                        treatmentBuilder.deferred().add(ins);
                     }
                     }
                 } else {
                 } else {
                     GroupKey key = soGrp.key;
                     GroupKey key = soGrp.key;
@@ -773,6 +766,12 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
         return rules;
         return rules;
     }
     }
 
 
+    /*
+     * Note: CpqD switches do not handle MPLS-related operation properly
+     * for a packet with VLAN tag. We pop VLAN here as a workaround.
+     * Side effect: HostService learns redundant hosts with same MAC but
+     * different VLAN. No known side effect on the network reachability.
+     */
     protected List<FlowRule> processEthDstFilter(EthCriterion ethCriterion,
     protected List<FlowRule> processEthDstFilter(EthCriterion ethCriterion,
                                        VlanIdCriterion vlanIdCriterion,
                                        VlanIdCriterion vlanIdCriterion,
                                        FilteringObjective filt,
                                        FilteringObjective filt,
@@ -783,12 +782,6 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
             vlanIdCriterion = (VlanIdCriterion) Criteria.matchVlanId(assignedVlan);
             vlanIdCriterion = (VlanIdCriterion) Criteria.matchVlanId(assignedVlan);
         }
         }
 
 
-        /*
-         * Note: CpqD switches do not handle MPLS-related operation properly
-         * for a packet with VLAN tag. We pop VLAN here as a workaround.
-         * Side effect: HostService learns redundant hosts with same MAC but
-         * different VLAN. No known side effect on the network reachability.
-         */
         List<FlowRule> rules = new ArrayList<>();
         List<FlowRule> rules = new ArrayList<>();
         TrafficSelector.Builder selectorIp = DefaultTrafficSelector
         TrafficSelector.Builder selectorIp = DefaultTrafficSelector
                 .builder();
                 .builder();