[CORD-46] Implement L2 switching in Segment Routing

DONE
- Update SpringOpenTTP to support bridging table emulation
- Populate low priority subnet broadcast entry for bridging table
- Move IP entry population to host event handler as well
- Update ArpHandler to handle intra-rack ARP forwarding/flooding
- Move TTL_OUT action from IP table to corresponding group
    Since hardware does not support TTL_OUT in the IP table
- Populate entries to bridging table (MAC learning)
- Emulate src mac table

Not in this submission
- Emulate src-mac table behavior
- Pop vlan in the group instead of the flow

Change-Id: Ib69357c1889ccddaa4daa272d9f5843790ee1a3c

@@ -20,10 +20,10 @@ import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.MacAddress;
+import org.onlab.packet.VlanId;
 import org.onosproject.net.ConnectPoint;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.Host;
-import org.onosproject.net.Port;
 import org.onosproject.net.PortNumber;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
@@ -60,12 +60,21 @@ public class ArpHandler {
 
     /**
      * Processes incoming ARP packets.
+     *
      * If it is an ARP request to router itself or known hosts,
      * then it sends ARP response.
      * If it is an ARP request to unknown hosts in its own subnet,
      * then it flood the ARP request to the ports.
      * If it is an ARP response, then set a flow rule for the host
      * and forward any IP packets to the host in the packet buffer to the host.
+     * <p>
+     * Note: We handles all ARP packet in, even for those ARP packets between
+     * hosts in the same subnet.
+     * For an ARP packet with broadcast destination MAC,
+     * some switches pipelines will send it to the controller due to table miss,
+     * other swithches will flood the packets directly in the data plane without
+     * packet in.
+     * We can deal with both cases.
      *
      * @param pkt incoming packet
      */
@@ -86,29 +95,56 @@ public class ArpHandler {
         if (arp.getOpCode() == ARP.OP_REQUEST) {
             handleArpRequest(deviceId, connectPoint, ethernet);
         } else {
-            srManager.ipHandler.forwardPackets(deviceId, hostIpAddress);
+            handleArpReply(deviceId, connectPoint, ethernet);
         }
     }
 
     private void handleArpRequest(DeviceId deviceId, ConnectPoint inPort, Ethernet payload) {
         ARP arpRequest = (ARP) payload.getPayload();
+        VlanId vlanId = VlanId.vlanId(payload.getVlanID());
         HostId targetHostId = HostId.hostId(MacAddress.valueOf(
-                arpRequest.getTargetHardwareAddress()));
+                                            arpRequest.getTargetHardwareAddress()),
+                                            vlanId);
 
-        // ARP request for router
+        // ARP request for router. Send ARP reply.
         if (isArpReqForRouter(deviceId, arpRequest)) {
             Ip4Address targetAddress = Ip4Address.valueOf(arpRequest.getTargetProtocolAddress());
-
-            sendArpResponse(arpRequest, config.getRouterMacForAGatewayIp(targetAddress));
+            sendArpResponse(arpRequest, config.getRouterMacForAGatewayIp(targetAddress), vlanId);
         } else {
             Host targetHost = srManager.hostService.getHost(targetHostId);
-            // ARP request for known hosts
+            // ARP request for known hosts. Send proxy ARP reply on behalf of the target.
             if (targetHost != null) {
-                sendArpResponse(arpRequest, targetHost.mac());
+                removeVlanAndForward(payload, targetHost.location());
+            // ARP request for unknown host in the subnet. Flood in the subnet.
+            } else {
+                removeVlanAndFlood(payload, inPort);
+            }
+        }
+    }
 
-            // ARP request for unknown host in the subnet
-            } else if (isArpReqForSubnet(deviceId, arpRequest)) {
-                flood(payload, inPort);
+    private void handleArpReply(DeviceId deviceId, ConnectPoint inPort, Ethernet payload) {
+        ARP arpReply = (ARP) payload.getPayload();
+        VlanId vlanId = VlanId.vlanId(payload.getVlanID());
+        HostId targetHostId = HostId.hostId(MacAddress.valueOf(
+                                            arpReply.getTargetHardwareAddress()),
+                                            vlanId);
+
+        // ARP reply for router. Process all pending IP packets.
+        if (isArpReqForRouter(deviceId, arpReply)) {
+            Ip4Address hostIpAddress = Ip4Address.valueOf(arpReply.getSenderProtocolAddress());
+            srManager.ipHandler.forwardPackets(deviceId, hostIpAddress);
+        } else {
+            Host targetHost = srManager.hostService.getHost(targetHostId);
+            // ARP reply for known hosts. Forward to the host.
+            if (targetHost != null) {
+                removeVlanAndForward(payload, targetHost.location());
+            // ARP reply for unknown host, Flood in the subnet.
+            } else {
+                // Don't flood to non-edge ports
+                if (vlanId.equals(VlanId.vlanId(srManager.ASSIGNED_VLAN_NO_SUBNET))) {
+                    return;
+                }
+                removeVlanAndFlood(payload, inPort);
             }
         }
     }
@@ -126,14 +162,6 @@ public class ArpHandler {
         return false;
     }
 
-    private boolean isArpReqForSubnet(DeviceId deviceId, ARP arpRequest) {
-        return config.getSubnets(deviceId).stream()
-                     .anyMatch((prefix)->
-                     prefix.contains(Ip4Address.
-                                     valueOf(arpRequest.
-                                             getTargetProtocolAddress())));
-    }
-
     /**
      * Sends an APR request for the target IP address to all ports except in-port.
      *
@@ -170,11 +198,10 @@ public class ArpHandler {
                 .setSourceMACAddress(senderMacAddress)
                 .setEtherType(Ethernet.TYPE_ARP).setPayload(arpRequest);
 
-        flood(eth, inPort);
+        removeVlanAndFlood(eth, inPort);
     }
 
-    private void sendArpResponse(ARP arpRequest, MacAddress targetMac) {
-
+    private void sendArpResponse(ARP arpRequest, MacAddress targetMac, VlanId vlanId) {
         ARP arpReply = new ARP();
         arpReply.setHardwareType(ARP.HW_TYPE_ETHERNET)
                 .setProtocolType(ARP.PROTO_TYPE_IP)
@@ -193,8 +220,9 @@ public class ArpHandler {
                 .setEtherType(Ethernet.TYPE_ARP).setPayload(arpReply);
 
 
-        HostId dstId = HostId.hostId(MacAddress.valueOf(
-                arpReply.getTargetHardwareAddress()));
+        HostId dstId = HostId.hostId(
+                MacAddress.valueOf(arpReply.getTargetHardwareAddress()),
+                vlanId);
         Host dst = srManager.hostService.getHost(dstId);
         if (dst == null) {
             log.warn("Cannot send ARP response to unknown device");
@@ -209,19 +237,51 @@ public class ArpHandler {
         srManager.packetService.emit(packet);
     }
 
-    private void flood(Ethernet request, ConnectPoint inPort) {
-        TrafficTreatment.Builder builder;
-        ByteBuffer buf = ByteBuffer.wrap(request.serialize());
-
-        for (Port port: srManager.deviceService.getPorts(inPort.deviceId())) {
-            if (!port.number().equals(inPort.port()) &&
-                    port.number().toLong() > 0) {
-                builder = DefaultTrafficTreatment.builder();
-                builder.setOutput(port.number());
-                srManager.packetService.emit(new DefaultOutboundPacket(inPort.deviceId(),
-                        builder.build(), buf));
-            }
+    /**
+     * Remove VLAN tag and flood to all ports in the same subnet.
+     *
+     * @param packet packet to be flooded
+     * @param inPort where the packet comes from
+     */
+    private void removeVlanAndFlood(Ethernet packet, ConnectPoint inPort) {
+        Ip4Address targetProtocolAddress = Ip4Address.valueOf(
+                ((ARP) packet.getPayload()).getTargetProtocolAddress()
+        );
+
+        srManager.deviceConfiguration.getSubnetPortsMap(inPort.deviceId()).forEach((subnet, ports) -> {
+            if (subnet.contains(targetProtocolAddress)) {
+                ports.stream()
+                        .filter(port -> port != inPort.port())
+                        .forEach(port -> {
+                            removeVlanAndForward(packet, new ConnectPoint(inPort.deviceId(), port));
+                        });
             }
+        });
     }
 
+    /**
+     * Remove VLAN tag and packet out to given port.
+     *
+     * Note: In current implementation, we expect all communication with
+     * end hosts within a subnet to be untagged.
+     * <p>
+     * For those pipelines that internally assigns a VLAN, the VLAN tag will be
+     * removed before egress.
+     * <p>
+     * For those pipelines that do not assign internal VLAN, the packet remains
+     * untagged.
+     *
+     * @param packet packet to be forwarded
+     * @param outPort where the packet should be forwarded
+     */
+    private void removeVlanAndForward(Ethernet packet, ConnectPoint outPort) {
+        packet.setEtherType(Ethernet.TYPE_ARP);
+        packet.setVlanID(Ethernet.VLAN_UNTAGGED);
+        ByteBuffer buf = ByteBuffer.wrap(packet.serialize());
+
+        TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
+        tbuilder.setOutput(outPort.port());
+        srManager.packetService.emit(new DefaultOutboundPacket(outPort.deviceId(),
+                                                               tbuilder.build(), buf));
+    }
 }

@@ -105,8 +105,17 @@ public class IcmpHandler {
         }
     }
 
+    /**
+     * Sends an ICMP reply message.
+     *
+     * Note: we assume that packets sending from the edge switches to the hosts
+     * have untagged VLAN.
+     * @param icmpRequest the original ICMP request
+     * @param outport the output port where the ICMP reply should be sent to
+     */
     private void sendICMPResponse(Ethernet icmpRequest, ConnectPoint outport) {
-
+        // Note: We assume that packets arrive at the edge switches have
+        // untagged VLAN.
         Ethernet icmpReplyEth = new Ethernet();
 
         IPv4 icmpRequestIpv4 = (IPv4) icmpRequest.getPayload();
@@ -129,7 +138,6 @@ public class IcmpHandler {
         icmpReplyEth.setEtherType(Ethernet.TYPE_IPV4);
         icmpReplyEth.setDestinationMACAddress(icmpRequest.getSourceMACAddress());
         icmpReplyEth.setSourceMACAddress(icmpRequest.getDestinationMACAddress());
-        icmpReplyEth.setVlanID(icmpRequest.getVlanID());
 
         Ip4Address destIpAddress = Ip4Address.valueOf(icmpReplyIpv4.getDestinationAddress());
         Ip4Address destRouterAddress = config.getRouterIpAddressForASubnetHost(destIpAddress);

@@ -55,7 +55,6 @@ import java.util.concurrent.atomic.AtomicLong;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 public class RoutingRulePopulator {
-
     private static final Logger log = LoggerFactory
             .getLogger(RoutingRulePopulator.class);
 
@@ -105,13 +104,45 @@ public class RoutingRulePopulator {
      */
     public void populateIpRuleForHost(DeviceId deviceId, Ip4Address hostIp,
                                       MacAddress hostMac, PortNumber outPort) {
-        MacAddress deviceMac;
+        log.debug("Populate IP table entry for host {} at {}:{}",
+                hostIp, deviceId, outPort);
+        ForwardingObjective.Builder fwdBuilder;
         try {
-            deviceMac = config.getDeviceMac(deviceId);
+            fwdBuilder = getForwardingObjectiveBuilder(
+                    deviceId, hostIp, hostMac, outPort);
         } catch (DeviceConfigNotFoundException e) {
             log.warn(e.getMessage() + " Aborting populateIpRuleForHost.");
             return;
         }
+        srManager.flowObjectiveService.
+            forward(deviceId, fwdBuilder.add(new SRObjectiveContext(deviceId,
+                    SRObjectiveContext.ObjectiveType.FORWARDING)));
+        rulePopulationCounter.incrementAndGet();
+    }
+
+    public void revokeIpRuleForHost(DeviceId deviceId, Ip4Address hostIp,
+            MacAddress hostMac, PortNumber outPort) {
+        log.debug("Revoke IP table entry for host {} at {}:{}",
+                hostIp, deviceId, outPort);
+        ForwardingObjective.Builder fwdBuilder;
+        try {
+            fwdBuilder = getForwardingObjectiveBuilder(
+                    deviceId, hostIp, hostMac, outPort);
+        } catch (DeviceConfigNotFoundException e) {
+            log.warn(e.getMessage() + " Aborting revokeIpRuleForHost.");
+            return;
+        }
+        srManager.flowObjectiveService.
+                forward(deviceId, fwdBuilder.remove(new SRObjectiveContext(deviceId,
+                        SRObjectiveContext.ObjectiveType.FORWARDING)));
+    }
+
+    private ForwardingObjective.Builder getForwardingObjectiveBuilder(
+            DeviceId deviceId, Ip4Address hostIp,
+            MacAddress hostMac, PortNumber outPort)
+            throws DeviceConfigNotFoundException {
+        MacAddress deviceMac;
+        deviceMac = config.getDeviceMac(deviceId);
 
         TrafficSelector.Builder sbuilder = DefaultTrafficSelector.builder();
         TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
@@ -127,19 +158,10 @@ public class RoutingRulePopulator {
         TrafficTreatment treatment = tbuilder.build();
         TrafficSelector selector = sbuilder.build();
 
-        ForwardingObjective.Builder fwdBuilder = DefaultForwardingObjective
-                .builder().fromApp(srManager.appId).makePermanent()
+        return DefaultForwardingObjective.builder()
+                .fromApp(srManager.appId).makePermanent()
                 .withSelector(selector).withTreatment(treatment)
                 .withPriority(100).withFlag(ForwardingObjective.Flag.SPECIFIC);
-
-        log.debug("Installing IPv4 forwarding objective "
-                + "for host {} in switch {}", hostIp, deviceId);
-        srManager.flowObjectiveService.
-            forward(deviceId,
-                    fwdBuilder.
-                    add(new SRObjectiveContext(deviceId,
-                                           SRObjectiveContext.ObjectiveType.FORWARDING)));
-        rulePopulationCounter.incrementAndGet();
     }
 
     /**
@@ -186,26 +208,25 @@ public class RoutingRulePopulator {
         }
 
         TrafficSelector.Builder sbuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
-
         sbuilder.matchIPDst(ipPrefix);
         sbuilder.matchEthType(Ethernet.TYPE_IPV4);
+        TrafficSelector selector = sbuilder.build();
 
-        NeighborSet ns = null;
+        TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
+        NeighborSet ns;
+        TrafficTreatment treatment;
 
         // If the next hop is the same as the final destination, then MPLS label
         // is not set.
         if (nextHops.size() == 1 && nextHops.toArray()[0].equals(destSw)) {
-            tbuilder.deferred().decNwTtl();
+            tbuilder.immediate().decNwTtl();
             ns = new NeighborSet(nextHops);
+            treatment = tbuilder.build();
         } else {
-            tbuilder.deferred().copyTtlOut();
             ns = new NeighborSet(nextHops, segmentId);
+            treatment = null;
         }
 
-        TrafficTreatment treatment = tbuilder.build();
-        TrafficSelector selector = sbuilder.build();
-
         if (srManager.getNextObjectiveId(deviceId, ns) <= 0) {
             log.warn("No next objective in {} for ns: {}", deviceId, ns);
             return false;
@@ -216,10 +237,12 @@ public class RoutingRulePopulator {
                 .fromApp(srManager.appId)
                 .makePermanent()
                 .nextStep(srManager.getNextObjectiveId(deviceId, ns))
-                .withTreatment(treatment)
                 .withSelector(selector)
                 .withPriority(100)
                 .withFlag(ForwardingObjective.Flag.SPECIFIC);
+        if (treatment != null) {
+            fwdBuilder.withTreatment(treatment);
+        }
         log.debug("Installing IPv4 forwarding objective "
                         + "for router IP/subnet {} in switch {}",
                 ipPrefix,
@@ -423,8 +446,6 @@ public class RoutingRulePopulator {
                         ? VlanId.vlanId(SegmentRoutingManager.ASSIGNED_VLAN_NO_SUBNET)
                         : srManager.getSubnetAssignedVlanId(deviceId, portSubnet);
 
-
-
                 FilteringObjective.Builder fob = DefaultFilteringObjective.builder();
                 fob.withKey(Criteria.matchInPort(port.number()))
                 .addCondition(Criteria.matchEthDst(deviceMac))
@@ -469,14 +490,14 @@ public class RoutingRulePopulator {
         Set<Ip4Address> allIps = new HashSet<Ip4Address>(config.getPortIPs(deviceId));
         allIps.add(routerIp);
         for (Ip4Address ipaddr : allIps) {
-            TrafficSelector.Builder selector = DefaultTrafficSelector.builder();
-            TrafficTreatment.Builder treatment = DefaultTrafficTreatment.builder();
-            selector.matchEthType(Ethernet.TYPE_IPV4);
-            selector.matchIPDst(IpPrefix.valueOf(ipaddr,
+            TrafficSelector.Builder sbuilder = DefaultTrafficSelector.builder();
+            TrafficTreatment.Builder tbuilder = DefaultTrafficTreatment.builder();
+            sbuilder.matchEthType(Ethernet.TYPE_IPV4);
+            sbuilder.matchIPDst(IpPrefix.valueOf(ipaddr,
                                                  IpPrefix.MAX_INET_MASK_LENGTH));
-            treatment.setOutput(PortNumber.CONTROLLER);
-            puntIp.withSelector(selector.build());
-            puntIp.withTreatment(treatment.build());
+            tbuilder.setOutput(PortNumber.CONTROLLER);
+            puntIp.withSelector(sbuilder.build());
+            puntIp.withTreatment(tbuilder.build());
             puntIp.withFlag(Flag.VERSATILE)
                 .withPriority(HIGHEST_PRIORITY)
                 .makePermanent()
@@ -489,6 +510,48 @@ public class RoutingRulePopulator {
         }
     }
 
+    /**
+     * Populates a forwarding objective to send packets that miss other high
+     * priority Bridging Table entries to a group that contains all ports of
+     * its subnet.
+     *
+     * Note: We assume that packets sending from the edge switches to the hosts
+     * have untagged VLAN.
+     * The VLAN tag will be popped later in the flooding group.
+     *
+     * @param deviceId switch ID to set the rules
+     */
+    public void populateSubnetBroadcastRule(DeviceId deviceId) {
+        config.getSubnets(deviceId).forEach(subnet -> {
+            int nextId = srManager.getSubnetNextObjectiveId(deviceId, subnet);
+            VlanId vlanId = srManager.getSubnetAssignedVlanId(deviceId, subnet);
+
+            /* Driver should treat objective with MacAddress.NONE as the
+             * subnet broadcast rule
+             */
+            TrafficSelector.Builder sbuilder = DefaultTrafficSelector.builder();
+            sbuilder.matchVlanId(vlanId);
+            sbuilder.matchEthDst(MacAddress.NONE);
+
+            ForwardingObjective.Builder fob = DefaultForwardingObjective.builder();
+            fob.withFlag(Flag.SPECIFIC)
+                    .withSelector(sbuilder.build())
+                    .nextStep(nextId)
+                    .withPriority(5)
+                    .fromApp(srManager.appId)
+                    .makePermanent();
+
+            srManager.flowObjectiveService.forward(
+                    deviceId,
+                    fob.add(new SRObjectiveContext(
+                                    deviceId,
+                                    SRObjectiveContext.ObjectiveType.FORWARDING)
+                    )
+            );
+        });
+    }
+
+
     private PortNumber selectOnePort(DeviceId srcId, Set<DeviceId> destIds) {
 
         Set<Link> links = srManager.linkService.getDeviceLinks(srcId);

@@ -224,8 +224,8 @@ public class DefaultGroupHandler {
                     .setEthSrc(nodeMacAddr);
             if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
                 tBuilder.pushMpls()
-                        .setMpls(MplsLabel.
-                                 mplsLabel(ns.getEdgeLabel()));
+                        .copyTtlOut()
+                        .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
             }
 
             Integer nextId = nsNextObjStore.
@@ -292,8 +292,9 @@ public class DefaultGroupHandler {
                     .setEthDst(dstMac)
                     .setEthSrc(nodeMacAddr);
             if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
-                tBuilder.pushMpls().setMpls(MplsLabel.mplsLabel(ns
-                                                    .getEdgeLabel()));
+                tBuilder.pushMpls()
+                        .copyTtlOut()
+                        .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
             }
 
             Integer nextId = nsNextObjStore.
@@ -536,8 +537,9 @@ public class DefaultGroupHandler {
                             .setEthDst(deviceMac)
                             .setEthSrc(nodeMacAddr);
                     if (ns.getEdgeLabel() != NeighborSet.NO_EDGE_LABEL) {
-                        tBuilder.pushMpls().setMpls(MplsLabel.mplsLabel(ns
-                                .getEdgeLabel()));
+                        tBuilder.pushMpls()
+                                .copyTtlOut()
+                                .setMpls(MplsLabel.mplsLabel(ns.getEdgeLabel()));
                     }
                     nextObjBuilder.addTreatment(tBuilder.build());
                 }

@@ -21,6 +21,7 @@ import java.util.List;
 
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.MacAddress;
+import org.onlab.packet.VlanId;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.net.behaviour.NextGroup;
 import org.onosproject.net.flow.DefaultFlowRule;
@@ -34,6 +35,7 @@ import org.onosproject.net.flow.criteria.EthCriterion;
 import org.onosproject.net.flow.criteria.EthTypeCriterion;
 import org.onosproject.net.flow.criteria.IPCriterion;
 import org.onosproject.net.flow.criteria.MplsCriterion;
+import org.onosproject.net.flow.criteria.VlanIdCriterion;
 import org.onosproject.net.flow.instructions.Instruction;
 import org.onosproject.net.flowobjective.FilteringObjective;
 import org.onosproject.net.flowobjective.ForwardingObjective;
@@ -175,12 +177,13 @@ public class SpringOpenTTPDell extends SpringOpenTTP {
     //Dell switches need ETH_DST based match condition in all IP table entries.
     //So while processing the ETH_DST based filtering objective, store
     //the device MAC to be used locally to use it while pushing the IP rules.
-    protected List<FlowRule> processEthDstFilter(Criterion c,
+    protected List<FlowRule> processEthDstFilter(EthCriterion ethCriterion,
+                                                 VlanIdCriterion vlanIdCriterion,
                                                  FilteringObjective filt,
+                                                 VlanId assignedVlan,
                                                  ApplicationId applicationId) {
         // Store device termination Mac to be used in IP flow entries
-        EthCriterion e = (EthCriterion) c;
-        deviceTMac = e.mac();
+        deviceTMac = ethCriterion.mac();
 
         log.debug("For now not adding any TMAC rules "
                 + "into Dell switches as it is ignoring");
@@ -189,8 +192,9 @@ public class SpringOpenTTPDell extends SpringOpenTTP {
     }
 
     @Override
-    protected List<FlowRule> processVlanIdFilter(Criterion c,
+    protected List<FlowRule> processVlanIdFilter(VlanIdCriterion vlanIdCriterion,
                                                  FilteringObjective filt,
+                                                 VlanId assignedVlan,
                                                  ApplicationId applicationId) {
         log.debug("For now not adding any VLAN rules "
                 + "into Dell switches as it is ignoring");