CORD-537 Added public IP gateways for ARP proxy

- Added public IP gateway and MAC pairs to network config for ARP proxy
- Added vSG as a ONOS host

Change-Id: Ia722ba3843297cec7134da5d64bbf188c22762f8

@@ -16,6 +16,7 @@
 package org.onosproject.cordvtn;
 
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
@@ -156,7 +157,7 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
     private HostProviderService hostProvider;
     private CordVtnRuleInstaller ruleInstaller;
     private CordVtnArpProxy arpProxy;
-    private volatile MacAddress gatewayMac = MacAddress.NONE;
+    private volatile MacAddress privateGatewayMac = MacAddress.NONE;
 
     /**
      * Creates an cordvtn host location provider.
@@ -305,7 +306,7 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
 
     @Override
     public void updateVirtualSubscriberGateways(HostId vSgHostId, String serviceVlan,
-                                                Set<IpAddress> vSgIps) {
+                                                Map<IpAddress, MacAddress> vSgs) {
         Host vSgVm = hostService.getHost(vSgHostId);
 
         if (vSgVm == null || !vSgVm.annotations().value(S_TAG).equals(serviceVlan)) {
@@ -313,8 +314,45 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
             return;
         }
 
-        log.info("Updates vSGs in {} with {}", vSgVm.id(), vSgIps.toString());
-        ruleInstaller.populateSubscriberGatewayRules(vSgVm, vSgIps);
+        log.info("Updates vSGs in {} with {}", vSgVm.id(), vSgs.toString());
+        vSgs.entrySet().stream()
+                .forEach(entry -> addVirtualSubscriberGateway(
+                        vSgVm,
+                        entry.getKey(),
+                        entry.getValue(),
+                        serviceVlan));
+
+        ruleInstaller.populateSubscriberGatewayRules(vSgVm, vSgs.keySet());
+    }
+
+    /**
+     * Adds virtual subscriber gateway to the system.
+     *
+     * @param vSgHost host virtual machine of this vSG
+     * @param vSgIp vSG ip address
+     * @param vSgMac vSG mac address
+     * @param serviceVlan service vlan
+     */
+    public void addVirtualSubscriberGateway(Host vSgHost, IpAddress vSgIp, MacAddress vSgMac, String serviceVlan) {
+        HostId hostId = HostId.hostId(vSgMac);
+        Host host = hostService.getHost(hostId);
+        if (host != null) {
+            log.debug("vSG with {} already exists", vSgMac.toString());
+            return;
+        }
+
+        log.info("vSG with IP({}) MAC({}) detected", vSgIp.toString(), vSgMac.toString());
+        DefaultAnnotations.Builder annotations = DefaultAnnotations.builder()
+                .set(S_TAG, serviceVlan);
+
+        HostDescription hostDesc = new DefaultHostDescription(
+                vSgMac,
+                VlanId.NONE,
+                vSgHost.location(),
+                Sets.newHashSet(vSgIp),
+                annotations.build());
+
+        hostProvider.hostDetected(hostId, hostDesc, false);
     }
 
     /**
@@ -438,24 +476,24 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
      * Returns public ip addresses of vSGs running inside a give vSG host.
      *
      * @param vSgHost vSG host
-     * @return set of ip address, or empty set
+     * @return map of ip and mac address, or empty map
      */
-    private Set<IpAddress> getSubscriberGatewayIps(Host vSgHost) {
+    private Map<IpAddress, MacAddress> getSubscriberGateways(Host vSgHost) {
         String vPortId = vSgHost.annotations().value(OPENSTACK_PORT_ID);
         String serviceVlan = vSgHost.annotations().value(S_TAG);
 
         OpenstackPort vPort = openstackService.port(vPortId);
         if (vPort == null) {
             log.warn("Failed to get OpenStack port {} for VM {}", vPortId, vSgHost.id());
-            return Sets.newHashSet();
+            return Maps.newHashMap();
         }
 
         if (!serviceVlan.equals(getServiceVlan(vPort))) {
             log.error("Host({}) s-tag does not match with vPort s-tag", vSgHost.id());
-            return Sets.newHashSet();
+            return Maps.newHashMap();
         }
 
-        return vPort.allowedAddressPairs().keySet();
+        return vPort.allowedAddressPairs();
     }
 
     /**
@@ -485,6 +523,11 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
      */
     private void serviceVmAdded(Host host) {
         String vNetId = host.annotations().value(SERVICE_ID);
+        if (vNetId == null) {
+            // ignore this host, it not a VM we injected or a vSG
+            return;
+        }
+
         OpenstackNetwork vNet = openstackService.network(vNetId);
         if (vNet == null) {
             log.warn("Failed to get OpenStack network {} for VM {}({}).",
@@ -509,19 +552,28 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
         } else {
             // TODO check if the service needs an update on its group buckets after done CORD-433
             ruleInstaller.updateServiceGroup(service);
-            arpProxy.addServiceIp(service.serviceIp());
+            arpProxy.addGateway(service.serviceIp(), privateGatewayMac);
 
             // sends gratuitous ARP here for the case of adding existing VMs
             // when ONOS or cordvtn app is restarted
-            arpProxy.sendGratuitousArp(service.serviceIp(), gatewayMac, Sets.newHashSet(host));
+            arpProxy.sendGratuitousArpForGateway(service.serviceIp(), Sets.newHashSet(host));
         }
 
         registerDhcpLease(host, service);
         ruleInstaller.populateBasicConnectionRules(host, getTunnelIp(host), vNet);
 
-        if (host.annotations().value(S_TAG) != null) {
+        String serviceVlan = host.annotations().value(S_TAG);
+        if (serviceVlan != null) {
             log.debug("vSG VM detected {}", host.id());
-            ruleInstaller.populateSubscriberGatewayRules(host, getSubscriberGatewayIps(host));
+            Map<IpAddress, MacAddress> vSgs = getSubscriberGateways(host);
+            vSgs.entrySet().stream()
+                    .forEach(entry -> addVirtualSubscriberGateway(
+                            host,
+                            entry.getKey(),
+                            entry.getValue(),
+                            serviceVlan));
+
+            ruleInstaller.populateSubscriberGatewayRules(host, vSgs.keySet());
         }
     }
 
@@ -566,7 +618,7 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
             ruleInstaller.updateServiceGroup(service);
 
             if (getHostsWithOpenstackNetwork(vNet).isEmpty()) {
-                arpProxy.removeServiceIp(service.serviceIp());
+                arpProxy.removeGateway(service.serviceIp());
             }
         }
     }
@@ -575,14 +627,17 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
      * Sets service network gateway MAC address and sends out gratuitous ARP to all
      * VMs to update the gateway MAC address.
      *
-     * @param mac mac address
+     * @param newMac mac address to update
      */
-    private void setServiceGatewayMac(MacAddress mac) {
-        if (mac != null && !mac.equals(gatewayMac)) {
-            gatewayMac = mac;
-            log.debug("Set service gateway MAC address to {}", gatewayMac.toString());
+    private void setPrivateGatewayMac(MacAddress newMac) {
+        if (newMac == null || newMac.equals(privateGatewayMac)) {
+            // no updates, do nothing
+            return;
         }
 
+        privateGatewayMac = newMac;
+        log.debug("Set service gateway MAC address to {}", privateGatewayMac.toString());
+
         // TODO get existing service list from XOS and replace the loop below
         Set<String> vNets = Sets.newHashSet();
         hostService.getHosts().forEach(host -> vNets.add(host.annotations().value(SERVICE_ID)));
@@ -591,12 +646,26 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
         vNets.stream().forEach(vNet -> {
             CordService service = getCordService(CordServiceId.of(vNet));
             if (service != null) {
-                arpProxy.sendGratuitousArp(
-                        service.serviceIp(),
-                        gatewayMac,
-                        service.hosts().keySet());
+                arpProxy.addGateway(service.serviceIp(), privateGatewayMac);
+                arpProxy.sendGratuitousArpForGateway(service.serviceIp(), service.hosts().keySet());
+            }
+        });
     }
+
+    /**
+     * Sets public gateway MAC address.
+     *
+     * @param publicGateways gateway ip and mac address pairs
+     */
+    private void setPublicGatewayMac(Map<IpAddress, MacAddress> publicGateways) {
+        publicGateways.entrySet()
+                .stream()
+                .forEach(entry -> {
+                    arpProxy.addGateway(entry.getKey(), entry.getValue());
+                    log.debug("Added public gateway IP {}, MAC {}",
+                              entry.getKey().toString(), entry.getValue().toString());
                 });
+        // TODO notice gateway MAC change to VMs holds this gateway IP
     }
 
     /**
@@ -609,7 +678,8 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
             return;
         }
 
-        setServiceGatewayMac(config.gatewayMac());
+        setPrivateGatewayMac(config.privateGatewayMac());
+        setPublicGatewayMac(config.publicGateways());
    }
 
     private class InternalHostListener implements HostListener {
@@ -644,7 +714,7 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
                 return;
             }
 
-            arpProxy.processArpPacket(context, ethPacket, gatewayMac);
+            arpProxy.processArpPacket(context, ethPacket);
         }
     }
 

@@ -15,7 +15,7 @@
  */
 package org.onosproject.cordvtn;
 
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
 import org.onlab.packet.ARP;
 import org.onlab.packet.EthType;
 import org.onlab.packet.Ethernet;
@@ -36,10 +36,10 @@ import org.onosproject.net.packet.PacketService;
 import org.slf4j.Logger;
 
 import java.nio.ByteBuffer;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 
-import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.slf4j.LoggerFactory.getLogger;
 
@@ -53,7 +53,7 @@ public class CordVtnArpProxy {
     private final PacketService packetService;
     private final HostService hostService;
 
-    private Set<Ip4Address> serviceIPs = Sets.newHashSet();
+    private final Map<Ip4Address, MacAddress> gateways = Maps.newConcurrentMap();
 
     /**
      * Default constructor.
@@ -96,23 +96,25 @@ public class CordVtnArpProxy {
     }
 
     /**
-     * Adds a given service IP address to be served.
+     * Adds a given gateway IP and MAC address to this ARP proxy.
      *
-     * @param serviceIp service ip
+     * @param gatewayIp gateway ip address
+     * @param gatewayMac gateway mac address
      */
-    public void addServiceIp(IpAddress serviceIp) {
-        checkNotNull(serviceIp);
-        serviceIPs.add(serviceIp.getIp4Address());
+    public void addGateway(IpAddress gatewayIp, MacAddress gatewayMac) {
+        checkNotNull(gatewayIp);
+        checkNotNull(gatewayMac);
+        gateways.put(gatewayIp.getIp4Address(), gatewayMac);
     }
 
     /**
      * Removes a given service IP address from this ARP proxy.
      *
-     * @param serviceIp service ip
+     * @param gatewayIp gateway ip address
      */
-    public void removeServiceIp(IpAddress serviceIp) {
-        checkNotNull(serviceIp);
-        serviceIPs.remove(serviceIp.getIp4Address());
+    public void removeGateway(IpAddress gatewayIp) {
+        checkNotNull(gatewayIp);
+        gateways.remove(gatewayIp.getIp4Address());
     }
 
     /**
@@ -123,27 +125,28 @@ public class CordVtnArpProxy {
      *
      * @param context packet context
      * @param ethPacket ethernet packet
-     * @param gatewayMac gateway mac address
      */
-    public void processArpPacket(PacketContext context, Ethernet ethPacket, MacAddress gatewayMac) {
+    public void processArpPacket(PacketContext context, Ethernet ethPacket) {
         ARP arpPacket = (ARP) ethPacket.getPayload();
         if (arpPacket.getOpCode() != ARP.OP_REQUEST) {
            return;
         }
 
         Ip4Address targetIp = Ip4Address.valueOf(arpPacket.getTargetProtocolAddress());
-        MacAddress macAddr = serviceIPs.contains(targetIp) ?
-                gatewayMac : getMacFromHostService(targetIp);
 
-        if (macAddr.equals(MacAddress.NONE)) {
+        MacAddress gatewayMac = gateways.get(targetIp);
+        MacAddress replyMac = gatewayMac != null ? gatewayMac : getMacFromHostService(targetIp);
+
+        if (replyMac.equals(MacAddress.NONE)) {
             log.debug("Failed to find MAC for {}", targetIp.toString());
             context.block();
             return;
         }
 
+        log.trace("Send ARP reply for {} with {}", targetIp.toString(), replyMac.toString());
         Ethernet ethReply = ARP.buildArpReply(
                 targetIp,
-                macAddr,
+                replyMac,
                 ethPacket);
 
         TrafficTreatment treatment = DefaultTrafficTreatment.builder()
@@ -161,14 +164,17 @@ public class CordVtnArpProxy {
     /**
      * Emits gratuitous ARP when a gateway mac address has been changed.
      *
-     * @param ip ip address to update MAC
-     * @param mac new mac address
+     * @param gatewayIp gateway ip address to update MAC
      * @param hosts set of hosts to send gratuitous ARP packet
      */
-    public void sendGratuitousArp(IpAddress ip, MacAddress mac, Set<Host> hosts) {
-        checkArgument(!mac.equals(MacAddress.NONE));
+    public void sendGratuitousArpForGateway(IpAddress gatewayIp, Set<Host> hosts) {
+        MacAddress gatewayMac = gateways.get(gatewayIp.getIp4Address());
+        if (gatewayMac == null) {
+            log.debug("Gateway {} is not registered to ARP proxy", gatewayIp.toString());
+            return;
+        }
 
-        Ethernet ethArp = buildGratuitousArp(ip.getIp4Address(), mac);
+        Ethernet ethArp = buildGratuitousArp(gatewayIp.getIp4Address(), gatewayMac);
         hosts.stream().forEach(host -> {
             TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                     .setOutput(host.location().port())

@@ -16,7 +16,9 @@
 package org.onosproject.cordvtn;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import org.onlab.packet.IpAddress;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.TpPort;
 import org.onosproject.core.ApplicationId;
@@ -24,6 +26,7 @@ import org.onosproject.net.DeviceId;
 import org.onosproject.net.config.Config;
 import org.slf4j.Logger;
 
+import java.util.Map;
 import java.util.Set;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -36,6 +39,9 @@ public class CordVtnConfig extends Config<ApplicationId> {
 
     protected final Logger log = getLogger(getClass());
 
+    public static final String PRIVATE_GATEWAY_MAC = "privateGatewayMac";
+    public static final String PUBLIC_GATEWAYS = "publicGateways";
+    public static final String GATEWAY_IP = "gatewayIp";
     public static final String GATEWAY_MAC = "gatewayMac";
     public static final String LOCAL_MANAGEMENT_IP = "localManagementIp";
     public static final String OVSDB_PORT = "ovsdbPort";
@@ -80,12 +86,12 @@ public class CordVtnConfig extends Config<ApplicationId> {
     }
 
     /**
-     * Returns gateway MAC address.
+     * Returns private network gateway MAC address.
      *
      * @return mac address, or null
      */
-    public MacAddress gatewayMac() {
-        JsonNode jsonNode = object.get(GATEWAY_MAC);
+    public MacAddress privateGatewayMac() {
+        JsonNode jsonNode = object.get(PRIVATE_GATEWAY_MAC);
         if (jsonNode == null) {
             return null;
         }
@@ -99,6 +105,31 @@ public class CordVtnConfig extends Config<ApplicationId> {
     }
 
     /**
+     * Returns public network gateway IP and MAC address pairs.
+     *
+     * @return map of ip and mac address
+     */
+    public Map<IpAddress, MacAddress> publicGateways() {
+        JsonNode jsonNodes = object.get(PUBLIC_GATEWAYS);
+        if (jsonNodes == null) {
+            return null;
+        }
+
+        Map<IpAddress, MacAddress> publicGateways = Maps.newHashMap();
+        jsonNodes.forEach(jsonNode -> {
+            try {
+                publicGateways.put(
+                        IpAddress.valueOf(jsonNode.path(GATEWAY_IP).asText()),
+                        MacAddress.valueOf(jsonNode.path(GATEWAY_MAC).asText()));
+            } catch (IllegalArgumentException | NullPointerException e) {
+                log.error("Wrong address format {}", e.toString());
+            }
+        });
+
+        return publicGateways;
+    }
+
+    /**
      * Returns local management network address.
      *
      * @return network address

@@ -16,10 +16,11 @@
 package org.onosproject.cordvtn;
 
 import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
 import org.onosproject.net.ConnectPoint;
 import org.onosproject.net.HostId;
 
-import java.util.Set;
+import java.util.Map;
 
 /**
  * Service for provisioning overlay virtual networks on compute nodes.
@@ -67,8 +68,8 @@ public interface CordVtnService {
      *
      * @param vSgHost host id of vSG host
      * @param serviceVlan service vlan id
-     * @param vSgIps set of ip address of vSGs running in this vSG host
+     * @param vSgs map of ip and mac address of vSGs running in this vSG host
      */
     void updateVirtualSubscriberGateways(HostId vSgHost, String serviceVlan,
-                                         Set<IpAddress> vSgIps);
+                                         Map<IpAddress, MacAddress> vSgs);
 }

@@ -17,7 +17,7 @@ package org.onosproject.cordvtn.rest;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.MacAddress;
 import org.onosproject.cordvtn.CordVtnService;
@@ -36,7 +36,7 @@ import javax.ws.rs.Produces;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import java.io.InputStream;
-import java.util.Set;
+import java.util.Map;
 
 
 /**
@@ -88,16 +88,17 @@ public class NeutronMl2PortsWebResource extends AbstractWebResource {
 
             // this is allowed address pairs updates
             MacAddress mac = MacAddress.valueOf(jsonNode.path(MAC_ADDRESS).asText());
-            Set<IpAddress> vSgIps = Sets.newHashSet();
+            Map<IpAddress, MacAddress> vSgs = Maps.newHashMap();
             jsonNode.path(ADDRESS_PAIRS).forEach(addrPair -> {
-                IpAddress ip = IpAddress.valueOf(addrPair.path(IP_ADDERSS).asText());
-                vSgIps.add(ip);
+                IpAddress pairIp = IpAddress.valueOf(addrPair.path(IP_ADDERSS).asText());
+                MacAddress pairMac = MacAddress.valueOf(addrPair.path(MAC_ADDRESS).asText());
+                vSgs.put(pairIp, pairMac);
             });
 
             service.updateVirtualSubscriberGateways(
                     HostId.hostId(mac),
                     name.substring(STAG_BEGIN_INDEX),
-                    vSgIps);
+                    vSgs);
         } catch (Exception e) {
             return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
         }