CORD-483 Made virtual network gateway MAC address configurable

- Added 'gatewayMAC' field to network config for cordvtn
- Implemented to send gratuitous ARP when gateway MAC is updated

Change-Id: I4f9050f4be64f04e0568515bbb95474513bbe057

@@ -36,6 +36,12 @@ import org.onosproject.net.HostId;
 import org.onosproject.net.HostLocation;
 import org.onosproject.net.Port;
 import org.onosproject.net.SparseAnnotations;
+import org.onosproject.net.config.ConfigFactory;
+import org.onosproject.net.config.NetworkConfigEvent;
+import org.onosproject.net.config.NetworkConfigListener;
+import org.onosproject.net.config.NetworkConfigRegistry;
+import org.onosproject.net.config.NetworkConfigService;
+import org.onosproject.net.config.basics.SubjectFactories;
 import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.driver.DriverService;
 import org.onosproject.net.flow.FlowRuleService;
@@ -62,10 +68,10 @@ import org.slf4j.Logger;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.stream.Collectors;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
 import static org.onlab.util.Tools.groupedThreads;
 import static org.slf4j.LoggerFactory.getLogger;
 
@@ -83,6 +89,12 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
     protected CoreService coreService;
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected NetworkConfigRegistry configRegistry;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected NetworkConfigService configService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected HostProviderRegistry hostProviderRegistry;
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
@@ -109,21 +121,31 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected OpenstackSwitchingService openstackService;
 
-    private static final int NUM_THREADS = 1;
+    private final ConfigFactory configFactory =
+            new ConfigFactory(SubjectFactories.APP_SUBJECT_FACTORY, CordVtnConfig.class, "cordvtn") {
+                @Override
+                public CordVtnConfig createConfig() {
+                    return new CordVtnConfig();
+                }
+            };
+
     private static final String DEFAULT_TUNNEL = "vxlan";
     private static final String SERVICE_ID = "serviceId";
     private static final String LOCATION_IP = "locationIp";
     private static final String OPENSTACK_VM_ID = "openstackVmId";
 
-    private final ExecutorService eventExecutor = Executors
-            .newFixedThreadPool(NUM_THREADS, groupedThreads("onos/cordvtn", "event-handler"));
+    private final ExecutorService eventExecutor =
+            newSingleThreadScheduledExecutor(groupedThreads("onos/cordvtn", "event-handler"));
 
     private final PacketProcessor packetProcessor = new InternalPacketProcessor();
     private final HostListener hostListener = new InternalHostListener();
+    private final NetworkConfigListener configListener = new InternalConfigListener();
 
+    private ApplicationId appId;
     private HostProviderService hostProvider;
     private CordVtnRuleInstaller ruleInstaller;
     private CordVtnArpProxy arpProxy;
+    private volatile MacAddress gatewayMac = MacAddress.NONE;
 
     /**
      * Creates an cordvtn host location provider.
@@ -134,8 +156,7 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
 
     @Activate
     protected void activate() {
-        ApplicationId appId = coreService.registerApplication("org.onosproject.cordvtn");
-
+        appId = coreService.registerApplication("org.onosproject.cordvtn");
         ruleInstaller = new CordVtnRuleInstaller(appId, flowRuleService,
                                                  deviceService,
                                                  driverService,
@@ -150,17 +171,24 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
         hostService.addListener(hostListener);
         hostProvider = hostProviderRegistry.register(this);
 
+        configRegistry.registerConfigFactory(configFactory);
+        configService.addListener(configListener);
+        readConfiguration();
+
         log.info("Started");
     }
 
     @Deactivate
     protected void deactivate() {
+        hostProviderRegistry.unregister(this);
         hostService.removeListener(hostListener);
+
         packetService.removeProcessor(packetProcessor);
 
-        eventExecutor.shutdown();
-        hostProviderRegistry.unregister(this);
+        configRegistry.unregisterConfigFactory(configFactory);
+        configService.removeListener(configListener);
 
+        eventExecutor.shutdown();
         log.info("Stopped");
     }
 
@@ -379,6 +407,10 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
             // TODO check if the service needs an update on its group buckets after done CORD-433
             ruleInstaller.updateServiceGroup(service);
             arpProxy.addServiceIp(service.serviceIp());
+
+            // sends gratuitous ARP here for the case of adding existing VMs
+            // when ONOS or cordvtn app is restarted
+            arpProxy.sendGratuitousArp(service.serviceIp(), gatewayMac, Sets.newHashSet(host));
         }
 
         ruleInstaller.populateBasicConnectionRules(host, getTunnelIp(host), vNet);
@@ -418,6 +450,47 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
         }
     }
 
+    /**
+     * Sets service network gateway MAC address and sends out gratuitous ARP to all
+     * VMs to update the gateway MAC address.
+     *
+     * @param mac mac address
+     */
+    private void setServiceGatewayMac(MacAddress mac) {
+        if (mac != null && !mac.equals(gatewayMac)) {
+            gatewayMac = mac;
+            log.debug("Set service gateway MAC address to {}", gatewayMac.toString());
+        }
+
+        // TODO get existing service list from XOS and replace the loop below
+        Set<String> vNets = Sets.newHashSet();
+        hostService.getHosts().forEach(host -> vNets.add(host.annotations().value(SERVICE_ID)));
+        vNets.remove(null);
+
+        vNets.stream().forEach(vNet -> {
+            CordService service = getCordService(CordServiceId.of(vNet));
+            if (service != null) {
+                arpProxy.sendGratuitousArp(
+                        service.serviceIp(),
+                        gatewayMac,
+                        service.hosts().keySet());
+            }
+        });
+    }
+
+    /**
+     * Updates configurations.
+     */
+    private void readConfiguration() {
+        CordVtnConfig config = configRegistry.getConfig(appId, CordVtnConfig.class);
+        if (config == null) {
+            log.debug("No configuration found");
+            return;
+        }
+
+        setServiceGatewayMac(config.gatewayMac());
+   }
+
     private class InternalHostListener implements HostListener {
 
         @Override
@@ -446,15 +519,31 @@ public class CordVtn extends AbstractProvider implements CordVtnService, HostPro
             }
 
             Ethernet ethPacket = context.inPacket().parsed();
-            if (ethPacket == null) {
+            if (ethPacket == null || ethPacket.getEtherType() != Ethernet.TYPE_ARP) {
                 return;
             }
 
-            if (ethPacket.getEtherType() != Ethernet.TYPE_ARP) {
+            arpProxy.processArpPacket(context, ethPacket, gatewayMac);
+        }
+    }
+
+    private class InternalConfigListener implements NetworkConfigListener {
+
+        @Override
+        public void event(NetworkConfigEvent event) {
+            if (!event.configClass().equals(CordVtnConfig.class)) {
                 return;
             }
 
-            arpProxy.processArpPacket(context, ethPacket);
+            switch (event.type()) {
+                case CONFIG_ADDED:
+                case CONFIG_UPDATED:
+                    log.info("Network configuration changed");
+                    eventExecutor.execute(CordVtn.this::readConfiguration);
+                    break;
+                default:
+                    break;
+            }
         }
     }
 }

@@ -23,6 +23,7 @@ import org.onlab.packet.Ip4Address;
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.MacAddress;
 import org.onosproject.core.ApplicationId;
+import org.onosproject.net.Host;
 import org.onosproject.net.flow.DefaultTrafficSelector;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
 import org.onosproject.net.flow.TrafficSelector;
@@ -37,6 +38,7 @@ import java.nio.ByteBuffer;
 import java.util.Optional;
 import java.util.Set;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.slf4j.LoggerFactory.getLogger;
 
@@ -45,8 +47,6 @@ import static org.slf4j.LoggerFactory.getLogger;
  */
 public class CordVtnArpProxy {
     protected final Logger log = getLogger(getClass());
-    // TODO make gateway MAC address configurable
-    private static final MacAddress DEFAULT_GATEWAY_MAC = MacAddress.valueOf("00:00:00:00:00:01");
 
     private final ApplicationId appId;
     private final PacketService packetService;
@@ -120,22 +120,21 @@ public class CordVtnArpProxy {
      *
      * @param context packet context
      * @param ethPacket ethernet packet
+     * @param gatewayMac gateway mac address
      */
-    public void processArpPacket(PacketContext context, Ethernet ethPacket) {
+    public void processArpPacket(PacketContext context, Ethernet ethPacket, MacAddress gatewayMac) {
+        checkArgument(!gatewayMac.equals(MacAddress.NONE));
+
         ARP arpPacket = (ARP) ethPacket.getPayload();
         Ip4Address targetIp = Ip4Address.valueOf(arpPacket.getTargetProtocolAddress());
 
-        if (arpPacket.getOpCode() != ARP.OP_REQUEST) {
+        if (arpPacket.getOpCode() != ARP.OP_REQUEST || !serviceIPs.contains(targetIp)) {
            return;
         }
 
-        if (!serviceIPs.contains(targetIp)) {
-            return;
-        }
-
         Ethernet ethReply = ARP.buildArpReply(
                 targetIp,
-                DEFAULT_GATEWAY_MAC,
+                gatewayMac,
                 ethPacket);
 
         TrafficTreatment treatment = DefaultTrafficTreatment.builder()
@@ -149,4 +148,57 @@ public class CordVtnArpProxy {
 
         context.block();
     }
+
+    /**
+     * Emits gratuitous ARP when a gateway mac address has been changed.
+     *
+     * @param ip ip address to update MAC
+     * @param mac new mac address
+     * @param hosts set of hosts to send gratuitous ARP packet
+     */
+    public void sendGratuitousArp(IpAddress ip, MacAddress mac, Set<Host> hosts) {
+        checkArgument(!mac.equals(MacAddress.NONE));
+
+        Ethernet ethArp = buildGratuitousArp(ip.getIp4Address(), mac);
+        hosts.stream().forEach(host -> {
+            TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+                    .setOutput(host.location().port())
+                    .build();
+
+            packetService.emit(new DefaultOutboundPacket(
+                    host.location().deviceId(),
+                    treatment,
+                    ByteBuffer.wrap(ethArp.serialize())));
+        });
+    }
+
+    /**
+     * Builds gratuitous ARP packet with a given IP and MAC address.
+     *
+     * @param ip ip address for TPA and SPA
+     * @param mac new mac address
+     * @return ethernet packet
+     */
+    private Ethernet buildGratuitousArp(IpAddress ip, MacAddress mac) {
+        Ethernet eth = new Ethernet();
+
+        eth.setEtherType(Ethernet.TYPE_ARP);
+        eth.setSourceMACAddress(mac);
+        eth.setDestinationMACAddress(MacAddress.BROADCAST);
+
+        ARP arp = new ARP();
+        arp.setOpCode(ARP.OP_REQUEST);
+        arp.setHardwareType(ARP.HW_TYPE_ETHERNET);
+        arp.setHardwareAddressLength((byte) Ethernet.DATALAYER_ADDRESS_LENGTH);
+        arp.setProtocolType(ARP.PROTO_TYPE_IP);
+        arp.setProtocolAddressLength((byte) Ip4Address.BYTE_LENGTH);
+
+        arp.setSenderHardwareAddress(mac.toBytes());
+        arp.setTargetHardwareAddress(MacAddress.BROADCAST.toBytes());
+        arp.setSenderProtocolAddress(ip.getIp4Address().toOctets());
+        arp.setTargetProtocolAddress(ip.getIp4Address().toOctets());
+
+        eth.setPayload(arp);
+        return eth;
+    }
 }

@@ -18,20 +18,25 @@ package org.onosproject.cordvtn;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.Sets;
 import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
 import org.onlab.packet.TpPort;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.config.Config;
+import org.slf4j.Logger;
 
 import java.util.Set;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static org.slf4j.LoggerFactory.getLogger;
 
 /**
  * Configuration object for CordVtn service.
  */
 public class CordVtnConfig extends Config<ApplicationId> {
 
+    protected final Logger log = getLogger(getClass());
+
     public static final String CORDVTN_NODES = "nodes";
     public static final String HOSTNAME = "hostname";
     public static final String OVSDB_IP = "ovsdbIp";
@@ -39,6 +44,7 @@ public class CordVtnConfig extends Config<ApplicationId> {
     public static final String BRIDGE_ID = "bridgeId";
     public static final String PHYSICAL_PORT_NAME = "phyPortName";
     public static final String LOCAL_IP = "localIp";
+    public static final String GATEWAY_MAC = "gatewayMac";
 
     /**
      * Returns the set of nodes read from network config.
@@ -64,6 +70,25 @@ public class CordVtnConfig extends Config<ApplicationId> {
     }
 
     /**
+     * Returns gateway MAC address.
+     *
+     * @return mac address, or null
+     */
+    public MacAddress gatewayMac() {
+        JsonNode jsonNode = object.get(GATEWAY_MAC);
+        if (jsonNode == null) {
+            return null;
+        }
+
+        try {
+            return MacAddress.valueOf(jsonNode.asText());
+        } catch (IllegalArgumentException e) {
+            log.error("Wrong MAC address format {}", jsonNode.asText());
+            return null;
+        }
+    }
+
+    /**
      * Configuration for CordVtn node.
      */
     public static class CordVtnNodeConfig {

@@ -41,12 +41,10 @@ import org.onosproject.net.behaviour.DefaultTunnelDescription;
 import org.onosproject.net.behaviour.TunnelConfig;
 import org.onosproject.net.behaviour.TunnelDescription;
 import org.onosproject.net.behaviour.TunnelName;
-import org.onosproject.net.config.ConfigFactory;
 import org.onosproject.net.config.NetworkConfigEvent;
 import org.onosproject.net.config.NetworkConfigListener;
 import org.onosproject.net.config.NetworkConfigRegistry;
 import org.onosproject.net.config.NetworkConfigService;
-import org.onosproject.net.config.basics.SubjectFactories;
 import org.onosproject.net.device.DeviceAdminService;
 import org.onosproject.net.device.DeviceEvent;
 import org.onosproject.net.device.DeviceListener;
@@ -151,14 +149,6 @@ public class CordVtnNodeManager {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected CordVtnService cordVtnService;
 
-    private final ConfigFactory configFactory =
-            new ConfigFactory(SubjectFactories.APP_SUBJECT_FACTORY, CordVtnConfig.class, "cordvtn") {
-                @Override
-                public CordVtnConfig createConfig() {
-                    return new CordVtnConfig();
-                }
-            };
-
     private final ExecutorService eventExecutor =
             newSingleThreadScheduledExecutor(groupedThreads("onos/cordvtncfg", "event-handler"));
 
@@ -231,7 +221,6 @@ public class CordVtnNodeManager {
     @Activate
     protected void active() {
         appId = coreService.getAppId(CordVtnService.CORDVTN_APP_ID);
-
         nodeStore = storageService.<CordVtnNode, NodeState>consistentMapBuilder()
                 .withSerializer(Serializer.using(NODE_SERIALIZER.build()))
                 .withName("cordvtn-nodestore")
@@ -247,12 +236,11 @@ public class CordVtnNodeManager {
 
         deviceService.addListener(deviceListener);
         configService.addListener(configListener);
-        configRegistry.registerConfigFactory(configFactory);
+        readConfiguration();
     }
 
     @Deactivate
     protected void deactivate() {
-        configRegistry.unregisterConfigFactory(configFactory);
         configService.removeListener(configListener);
         deviceService.removeListener(deviceListener);
 
@@ -820,13 +808,13 @@ public class CordVtnNodeManager {
     }
 
     /**
-     * Reads node configuration from config file.
+     * Reads cordvtn nodes from config file.
      */
     private void readConfiguration() {
         CordVtnConfig config = configRegistry.getConfig(appId, CordVtnConfig.class);
 
         if (config == null) {
-            log.warn("No configuration found");
+            log.debug("No configuration found");
             return;
         }
 
@@ -841,6 +829,8 @@ public class CordVtnNodeManager {
 
             addNode(cordVtnNode);
         });
+
+        // TODO remove nodes if needed
     }
 
     private class InternalConfigListener implements NetworkConfigListener {
@@ -853,11 +843,7 @@ public class CordVtnNodeManager {
 
             switch (event.type()) {
                 case CONFIG_ADDED:
-                    log.info("Network configuration added");
-                    eventExecutor.execute(CordVtnNodeManager.this::readConfiguration);
-                    break;
                 case CONFIG_UPDATED:
-                    log.info("Network configuration updated");
                     eventExecutor.execute(CordVtnNodeManager.this::readConfiguration);
                     break;
                 default: