Refactored OpenstackRouting to support multiple gateway nodes

Change-Id: I6870ca9a4fd6f6b1cf2d2be72f52ef87827e1d2c

@@ -20,7 +20,7 @@ import java.util.Objects;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 
 /**
 /**
- * An Openstack Neutron Router Interface Model.
+ * An OpenStack Neutron router interface model.
  */
  */
 public final class OpenstackRouterInterface {
 public final class OpenstackRouterInterface {
     private final String id;
     private final String id;
@@ -37,9 +37,9 @@ public final class OpenstackRouterInterface {
     }
     }
 
 
     /**
     /**
-     * Returns Router Interface ID.
+     * Returns router interface ID.
      *
      *
-     * @return router interface ID
+     * @return router interface id
      */
      */
     public String id() {
     public String id() {
         return id;
         return id;
@@ -48,7 +48,7 @@ public final class OpenstackRouterInterface {
     /**
     /**
      * Returns tenant ID.
      * Returns tenant ID.
      *
      *
-     * @return tenant ID
+     * @return tenant id
      */
      */
     public String tenantId() {
     public String tenantId() {
         return tenantId;
         return tenantId;
@@ -57,7 +57,7 @@ public final class OpenstackRouterInterface {
     /**
     /**
      * Returns subnet ID.
      * Returns subnet ID.
      *
      *
-     * @return subnet ID
+     * @return subnet id
      */
      */
     public String subnetId() {
     public String subnetId() {
         return subnetId;
         return subnetId;
@@ -66,7 +66,7 @@ public final class OpenstackRouterInterface {
     /**
     /**
      * Returns port ID.
      * Returns port ID.
      *
      *
-     * @return port ID
+     * @return port id
      */
      */
     public String portId() {
     public String portId() {
         return portId;
         return portId;
@@ -96,7 +96,16 @@ public final class OpenstackRouterInterface {
     }
     }
 
 
     /**
     /**
-     * An Openstack Router Interface Builder class.
+     * Returns OpenStack router interface builder.
+     *
+     * @return openstack router interface builder
+     */
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * An OpenStack Router interface builder class.
      */
      */
     public static final class Builder {
     public static final class Builder {
         private String id;
         private String id;
@@ -105,10 +114,10 @@ public final class OpenstackRouterInterface {
         private String portId;
         private String portId;
 
 
         /**
         /**
-         * Sets Router Interface ID.
+         * Sets router interface ID.
          *
          *
-         * @param id router interface ID
-         * @return Builder object
+         * @param id router interface id
+         * @return builder object
          */
          */
         public Builder id(String id) {
         public Builder id(String id) {
             this.id = id;
             this.id = id;
@@ -119,7 +128,7 @@ public final class OpenstackRouterInterface {
          * Sets tenant ID.
          * Sets tenant ID.
          *
          *
          * @param tenantId tenant ID
          * @param tenantId tenant ID
-         * @return Builder object
+         * @return builder object
          */
          */
         public Builder tenantId(String tenantId) {
         public Builder tenantId(String tenantId) {
             this.tenantId = tenantId;
             this.tenantId = tenantId;
@@ -130,7 +139,7 @@ public final class OpenstackRouterInterface {
          * Sets subnet ID.
          * Sets subnet ID.
          *
          *
          * @param subnetId subnet ID
          * @param subnetId subnet ID
-         * @return Builder object
+         * @return builder object
          */
          */
         public Builder subnetId(String subnetId) {
         public Builder subnetId(String subnetId) {
             this.subnetId = subnetId;
             this.subnetId = subnetId;
@@ -141,7 +150,7 @@ public final class OpenstackRouterInterface {
          * Sets port ID.
          * Sets port ID.
          *
          *
          * @param portId port ID
          * @param portId port ID
-         * @return Builder object
+         * @return builder object
          */
          */
         public Builder portId(String portId) {
         public Builder portId(String portId) {
             this.portId = portId;
             this.portId = portId;
@@ -149,14 +158,13 @@ public final class OpenstackRouterInterface {
         }
         }
 
 
         /**
         /**
-         * Builds an Openstack Router Interface object.
+         * Builds an OpenStack router interface object.
          *
          *
-         * @return OpenstackRouterInterface object
+         * @return openstack router interface object
          */
          */
         public OpenstackRouterInterface build() {
         public OpenstackRouterInterface build() {
             return new OpenstackRouterInterface(checkNotNull(id), checkNotNull(tenantId),
             return new OpenstackRouterInterface(checkNotNull(id), checkNotNull(tenantId),
                     checkNotNull(subnetId), checkNotNull(portId));
                     checkNotNull(subnetId), checkNotNull(portId));
         }
         }
-
     }
     }
 }
 }

@@ -13,20 +13,18 @@
  * See the License for the specific language governing permissions and
  * See the License for the specific language governing permissions and
  * limitations under the License.
  * limitations under the License.
  */
  */
-package org.onosproject.openstacknetworking.switching;
+package org.onosproject.openstacknetworking;
 
 
 import org.onlab.osgi.DefaultServiceDirectory;
 import org.onlab.osgi.DefaultServiceDirectory;
 import org.onlab.osgi.ServiceDirectory;
 import org.onlab.osgi.ServiceDirectory;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.Ip4Address;
 import org.onlab.util.Tools;
 import org.onlab.util.Tools;
-import org.onosproject.core.ApplicationId;
 import org.onosproject.core.CoreService;
 import org.onosproject.core.CoreService;
 import org.onosproject.mastership.MastershipService;
 import org.onosproject.mastership.MastershipService;
 import org.onosproject.net.Host;
 import org.onosproject.net.Host;
 import org.onosproject.net.host.HostEvent;
 import org.onosproject.net.host.HostEvent;
 import org.onosproject.net.host.HostListener;
 import org.onosproject.net.host.HostListener;
 import org.onosproject.net.host.HostService;
 import org.onosproject.net.host.HostService;
-import org.onosproject.openstacknetworking.Constants;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
 import java.util.Objects;
 import java.util.Objects;
@@ -53,7 +51,6 @@ public abstract class AbstractVmHandler {
     protected MastershipService mastershipService;
     protected MastershipService mastershipService;
     protected HostService hostService;
     protected HostService hostService;
 
 
-    protected ApplicationId appId;
     protected HostListener hostListener = new InternalHostListener();
     protected HostListener hostListener = new InternalHostListener();
 
 
     protected void activate() {
     protected void activate() {
@@ -61,8 +58,6 @@ public abstract class AbstractVmHandler {
         coreService = services.get(CoreService.class);
         coreService = services.get(CoreService.class);
         mastershipService = services.get(MastershipService.class);
         mastershipService = services.get(MastershipService.class);
         hostService = services.get(HostService.class);
         hostService = services.get(HostService.class);
-
-        appId = coreService.registerApplication(Constants.APP_ID);
         hostService.addListener(hostListener);
         hostService.addListener(hostListener);
 
 
         log.info("Started");
         log.info("Started");
@@ -75,9 +70,19 @@ public abstract class AbstractVmHandler {
         log.info("Stopped");
         log.info("Stopped");
     }
     }
 
 
-    abstract void hostDetected(Host host);
-
-    abstract void hostRemoved(Host host);
+    /**
+     * Performs any action when a host is detected.
+     *
+     * @param host detected host
+     */
+    protected abstract void hostDetected(Host host);
+
+    /**
+     * Performs any action when a host is removed.
+     *
+     * @param host removed host
+     */
+    protected abstract void hostRemoved(Host host);
 
 
     protected boolean isValidHost(Host host) {
     protected boolean isValidHost(Host host) {
         return !host.ipAddresses().isEmpty() &&
         return !host.ipAddresses().isEmpty() &&

@@ -28,23 +28,27 @@ public final class Constants {
     private Constants() {
     private Constants() {
     }
     }
 
 
-    public static final String APP_ID = "org.onosproject.openstackswitching";
+    public static final String SWITCHING_APP_ID = "org.onosproject.openstackswitching";
+    public static final String ROUTING_APP_ID = "org.onosproject.openstackrouting";
 
 
-    public static final String PORTNAME_PREFIX_VM = "tap";
-    public static final String PORTNAME_PREFIX_ROUTER = "qr-";
-    public static final String PORTNAME_PREFIX_TUNNEL = "vxlan";
+    public static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
+    public static final String DEVICE_OWNER_ROUTER_GATEWAY = "network:router_gateway";
+    public static final String DEVICE_OWNER_FLOATING_IP = "network:floatingip";
 
 
-    public static final MacAddress GATEWAY_MAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f");
+    public static final String PORT_NAME_PREFIX_VM = "tap";
+    public static final String PORT_NAME_PREFIX_TUNNEL = "vxlan";
 
 
-    // TODO: Please change these valuses following the way vrouter is implemented
-    public static final MacAddress GW_EXT_INT_MAC = MacAddress.valueOf("56:e6:30:a6:8c:e5");
-    public static final MacAddress PHY_ROUTER_MAC = MacAddress.valueOf("00:00:00:00:01:01");
+    public static final String DEFAULT_GATEWAY_MAC_STR = "fe:00:00:00:00:02";
+    public static final MacAddress DEFAULT_GATEWAY_MAC = MacAddress.valueOf(DEFAULT_GATEWAY_MAC_STR);
+    // TODO make this configurable
+    public static final MacAddress DEFAULT_EXTERNAL_ROUTER_MAC = MacAddress.valueOf("fe:00:00:00:00:01");
 
 
     public static final Ip4Address DNS_SERVER_IP = Ip4Address.valueOf("8.8.8.8");
     public static final Ip4Address DNS_SERVER_IP = Ip4Address.valueOf("8.8.8.8");
     public static final IpPrefix IP_PREFIX_ANY = Ip4Prefix.valueOf("0.0.0.0/0");
     public static final IpPrefix IP_PREFIX_ANY = Ip4Prefix.valueOf("0.0.0.0/0");
     public static final int DHCP_INFINITE_LEASE = -1;
     public static final int DHCP_INFINITE_LEASE = -1;
 
 
     public static final String NETWORK_ID = "networkId";
     public static final String NETWORK_ID = "networkId";
+    public static final String SUBNET_ID = "subnetId";
     public static final String PORT_ID = "portId";
     public static final String PORT_ID = "portId";
     public static final String VXLAN_ID = "vxlanId";
     public static final String VXLAN_ID = "vxlanId";
     public static final String TENANT_ID = "tenantId";
     public static final String TENANT_ID = "tenantId";
@@ -55,4 +59,8 @@ public final class Constants {
     public static final int TUNNELTAG_RULE_PRIORITY = 30000;
     public static final int TUNNELTAG_RULE_PRIORITY = 30000;
     public static final int ACL_RULE_PRIORITY = 30000;
     public static final int ACL_RULE_PRIORITY = 30000;
 
 
+    public static final int ROUTING_RULE_PRIORITY = 25000;
+    public static final int FLOATING_RULE_PRIORITY = 42000;
+    public static final int PNAT_RULE_PRIORITY = 26000;
+    public static final int PNAT_TIMEOUT = 120;
 }
 }
\ No newline at end of file

+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking;
+
+import org.onosproject.openstackinterface.OpenstackFloatingIP;
+
+/**
+ * Handles floating IP update requests from OpenStack.
+ */
+public interface OpenstackFloatingIpService {
+
+    enum Action {
+        ASSOCIATE,
+        DISSASSOCIATE
+    }
+
+    /**
+     * Handles floating IP create request from OpenStack.
+     *
+     * @param floatingIp floating IP information
+     */
+    void createFloatingIp(OpenstackFloatingIP floatingIp);
+
+    /**
+     * Handles floating IP update request from OpenStack.
+     *
+     * @param floatingIp floating IP information
+     */
+    void updateFloatingIp(OpenstackFloatingIP floatingIp);
+
+    /**
+     * Handles floating IP remove request from OpenStack.
+     *
+     * @param floatingIpId floating ip identifier
+     */
+    void deleteFloatingIp(String floatingIpId);
+}

@@ -15,76 +15,49 @@
  */
  */
 package org.onosproject.openstacknetworking;
 package org.onosproject.openstacknetworking;
 
 
-import org.onosproject.openstackinterface.OpenstackFloatingIP;
 import org.onosproject.openstackinterface.OpenstackRouter;
 import org.onosproject.openstackinterface.OpenstackRouter;
 import org.onosproject.openstackinterface.OpenstackRouterInterface;
 import org.onosproject.openstackinterface.OpenstackRouterInterface;
 
 
 /**
 /**
- * Supports L3 management REST API for openstack.
+ * Handles router update requests from OpenStack.
  */
  */
 public interface OpenstackRoutingService {
 public interface OpenstackRoutingService {
 
 
     /**
     /**
-     * Stores the floating IP information created by openstack.
+     * Handles the router create request from OpenStack.
      *
      *
-     * @param openstackFloatingIp Floating IP information
+     * @param osRouter router information
      */
      */
-    void createFloatingIP(OpenstackFloatingIP openstackFloatingIp);
+    void createRouter(OpenstackRouter osRouter);
 
 
     /**
     /**
-     * Updates flow rules corresponding to the floating IP information updated by openstack.
+     * Handles the router update request from OpenStack.
+     * Update router is called when the name, administrative state, or the external
+     * gateway setting is updated. The external gateway update is the only case
+     * that openstack routing service cares.
      *
      *
-     * @param openstackFloatingIp Floating IP information
+     * @param osRouter router information
      */
      */
-    void updateFloatingIP(OpenstackFloatingIP openstackFloatingIp);
+    void updateRouter(OpenstackRouter osRouter);
 
 
     /**
     /**
-     * Removes flow rules corresponding to floating IP information removed by openstack.
+     * Handles the router remove request from OpenStack.
      *
      *
-     * @param id Deleted Floating IP`s ID
+     * @param osRouterId identifier of the router
      */
      */
-    void deleteFloatingIP(String id);
+    void removeRouter(String osRouterId);
 
 
     /**
     /**
-     * Stores the router information created by openstack.
+     * Handles router interface add request from OpenStack.
      *
      *
-     * @param openstackRouter Router information
+     * @param osInterface router interface information
      */
      */
-    void createRouter(OpenstackRouter openstackRouter);
+    void addRouterInterface(OpenstackRouterInterface osInterface);
 
 
     /**
     /**
-     * Updates flow rules corresponding to the router information updated by openstack.
+     * Handles router interface remove request from OpenStack.
      *
      *
-     * @param openstackRouter Router information
+     * @param osInterface router interface information
      */
      */
-    void updateRouter(OpenstackRouter openstackRouter);
-
-    /**
-     * Removes flow rules corresponding to the router information removed by openstack.
-     *
-     * @param id Deleted router`s ID
-     */
-    void deleteRouter(String id);
-
-    /**
-     * Updates flow rules corresponding to the router information updated by openstack.
-     *
-     * @param openstackRouterInterface Router interface information
-     */
-    void updateRouterInterface(OpenstackRouterInterface openstackRouterInterface);
-
-    /**
-     * Removes flow rules corresponding to the router information removed by openstack.
-     *
-     * @param openstackRouterInterface Router interface information
-     */
-    void removeRouterInterface(OpenstackRouterInterface openstackRouterInterface);
-
-    /**
-     * Returns network id for routerInterface.
-     *
-     * @param portId routerInterface`s port id
-     * @return network id
-     */
-    String networkIdForRouterInterface(String portId);
+    void removeRouterInterface(OpenstackRouterInterface osInterface);
 }
 }

+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking;
+
+import org.onlab.packet.Ip4Address;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.net.Device;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.instructions.ExtensionPropertyException;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
+import org.onosproject.net.flowobjective.DefaultForwardingObjective;
+import org.onosproject.net.flowobjective.FlowObjectiveService;
+import org.onosproject.net.flowobjective.ForwardingObjective;
+import org.slf4j.Logger;
+
+import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Provides common methods to help populating flow rules for SONA applications.
+ */
+public final class RulePopulatorUtil {
+
+    protected static final Logger log = getLogger(RulePopulatorUtil.class);
+
+    private static final String TUNNEL_DST = "tunnelDst";
+
+    private RulePopulatorUtil() {
+    }
+
+    /**
+     * Returns tunnel destination extension treatment object.
+     *
+     * @param deviceService driver service
+     * @param deviceId device id to apply this treatment
+     * @param remoteIp tunnel destination ip address
+     * @return extension treatment
+     */
+    public static ExtensionTreatment buildExtension(DeviceService deviceService,
+                                                    DeviceId deviceId,
+                                                    Ip4Address remoteIp) {
+        Device device = deviceService.getDevice(deviceId);
+        if (device != null && !device.is(ExtensionTreatmentResolver.class)) {
+            log.error("The extension treatment is not supported");
+            return null;
+        }
+
+        ExtensionTreatmentResolver resolver = device.as(ExtensionTreatmentResolver.class);
+        ExtensionTreatment treatment = resolver.getExtensionInstruction(NICIRA_SET_TUNNEL_DST.type());
+        try {
+            treatment.setPropertyValue(TUNNEL_DST, remoteIp);
+            return treatment;
+        } catch (ExtensionPropertyException e) {
+            log.warn("Failed to get tunnelDst extension treatment for {}", deviceId);
+            return null;
+        }
+    }
+
+    /**
+     * Removes flow rules with the supplied information.
+     *
+     * @param flowObjectiveService flow objective service
+     * @param appId application id
+     * @param deviceId device id to remove this flow rule
+     * @param selector traffic selector
+     * @param flag flag
+     * @param priority priority
+     */
+    public static void removeRule(FlowObjectiveService flowObjectiveService,
+                           ApplicationId appId,
+                           DeviceId deviceId,
+                           TrafficSelector selector,
+                           ForwardingObjective.Flag flag,
+                           int priority) {
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(selector)
+                .withTreatment(DefaultTrafficTreatment.builder().build())
+                .withFlag(flag)
+                .withPriority(priority)
+                .fromApp(appId)
+                .remove();
+
+        flowObjectiveService.forward(deviceId, fo);
+    }
+}

-/*
- * Copyright 2016-present Open Networking Laboratory
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.onosproject.openstacknetworking.routing;
-
-import org.onosproject.net.Host;
-import org.onosproject.openstackinterface.OpenstackFloatingIP;
-
-/**
- * Handle FloatingIP Event for Managing Flow Rules In Openstack Nodes.
- */
-public class OpenstackFloatingIPHandler implements Runnable {
-
-    public enum Action {
-        ASSOCIATE,
-        DISSASSOCIATE
-    }
-
-    private final OpenstackFloatingIP floatingIP;
-    private final OpenstackRoutingRulePopulator rulePopulator;
-    private final Host host;
-    private final Action action;
-
-
-    OpenstackFloatingIPHandler(OpenstackRoutingRulePopulator rulePopulator,
-                               OpenstackFloatingIP openstackFloatingIP, Action action, Host host) {
-        this.floatingIP = openstackFloatingIP;
-        this.rulePopulator = rulePopulator;
-        this.action = action;
-        this.host = host;
-    }
-
-    @Override
-    public void run() {
-        if (action == Action.ASSOCIATE) {
-            rulePopulator.populateFloatingIpRules(floatingIP);
-        } else {
-            rulePopulator.removeFloatingIpRules(floatingIP, host);
-        }
-    }
-}

+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import com.google.common.base.Strings;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IpAddress;
+import org.onlab.util.KryoNamespace;
+import org.onlab.util.Tools;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Host;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flowobjective.DefaultForwardingObjective;
+import org.onosproject.net.flowobjective.FlowObjectiveService;
+import org.onosproject.net.flowobjective.ForwardingObjective;
+import org.onosproject.net.host.HostService;
+import org.onosproject.openstackinterface.OpenstackFloatingIP;
+import org.onosproject.openstacknetworking.Constants;
+import org.onosproject.openstacknetworking.OpenstackFloatingIpService;
+import org.onosproject.openstacknetworking.RulePopulatorUtil;
+import org.onosproject.openstacknode.OpenstackNode;
+import org.onosproject.openstacknode.OpenstackNodeEvent;
+import org.onosproject.openstacknode.OpenstackNodeListener;
+import org.onosproject.openstacknode.OpenstackNodeService;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
+import org.onosproject.store.service.Versioned;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.openstacknetworking.Constants.*;
+import static org.onosproject.openstacknetworking.RulePopulatorUtil.buildExtension;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.GATEWAY;
+
+
+@Service
+@Component(immediate = true)
+public class OpenstackFloatingIpManager implements OpenstackFloatingIpService {
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected CoreService coreService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected HostService hostService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DeviceService deviceService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected FlowObjectiveService flowObjectiveService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected StorageService storageService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected OpenstackNodeService nodeService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ScalableGatewayService gatewayService;
+
+    private static final String NOT_ASSOCIATED = "null";
+    private static final KryoNamespace.Builder FLOATING_IP_SERIALIZER =
+            KryoNamespace.newBuilder().register(KryoNamespaces.API);
+
+    private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(
+            groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+    private final InternalNodeListener nodeListener = new InternalNodeListener();
+    private ConsistentMap<IpAddress, Host> floatingIpMap;
+
+    private ApplicationId appId;
+
+    @Activate
+    protected void activate() {
+        appId = coreService.registerApplication(ROUTING_APP_ID);
+        nodeService.addListener(nodeListener);
+        floatingIpMap = storageService.<IpAddress, Host>consistentMapBuilder()
+                .withSerializer(Serializer.using(FLOATING_IP_SERIALIZER.build()))
+                .withName("openstackrouting-floatingip")
+                .withApplicationId(appId)
+                .build();
+
+        log.info("Started");
+    }
+
+    @Deactivate
+    protected void deactivate() {
+        nodeService.removeListener(nodeListener);
+        log.info("Stopped");
+    }
+
+    @Override
+    public void createFloatingIp(OpenstackFloatingIP floatingIp) {
+    }
+
+    @Override
+    public void updateFloatingIp(OpenstackFloatingIP floatingIp) {
+        if (Strings.isNullOrEmpty(floatingIp.portId()) ||
+                floatingIp.portId().equals(NOT_ASSOCIATED)) {
+            eventExecutor.execute(() -> disassociateFloatingIp(floatingIp));
+        } else {
+            eventExecutor.execute(() -> associateFloatingIp(floatingIp));
+        }
+    }
+
+    @Override
+    public void deleteFloatingIp(String floatingIpId) {
+    }
+
+    private void associateFloatingIp(OpenstackFloatingIP floatingIp) {
+        Optional<Host> associatedVm = Tools.stream(hostService.getHosts())
+                .filter(host -> Objects.equals(
+                        host.annotations().value(PORT_ID),
+                        floatingIp.portId()))
+                .findAny();
+        if (!associatedVm.isPresent()) {
+            log.warn("Failed to associate floating IP({}) to port:{}",
+                     floatingIp.floatingIpAddress(),
+                     floatingIp.portId());
+            return;
+        }
+
+        floatingIpMap.put(floatingIp.floatingIpAddress(), associatedVm.get());
+        populateFloatingIpRules(floatingIp.floatingIpAddress(), associatedVm.get());
+
+        log.info("Associated floating IP {} to fixed IP {}",
+                 floatingIp.floatingIpAddress(), floatingIp.fixedIpAddress());
+    }
+
+    private void disassociateFloatingIp(OpenstackFloatingIP floatingIp) {
+        Versioned<Host> associatedVm = floatingIpMap.remove(floatingIp.floatingIpAddress());
+        if (associatedVm == null) {
+            log.warn("Failed to disassociate floating IP({})",
+                     floatingIp.floatingIpAddress());
+            // No VM is actually associated with the floating IP, do nothing
+            return;
+        }
+
+        removeFloatingIpRules(floatingIp.floatingIpAddress(), associatedVm.value());
+        log.info("Disassociated floating IP {} from fixed IP {}",
+                 floatingIp.floatingIpAddress(),
+                 associatedVm.value().ipAddresses());
+    }
+
+    private void populateFloatingIpRules(IpAddress floatingIp, Host associatedVm) {
+        populateFloatingIpIncomingRules(floatingIp, associatedVm);
+        populateFloatingIpOutgoingRules(floatingIp, associatedVm);
+    }
+
+    private void removeFloatingIpRules(IpAddress floatingIp, Host associatedVm) {
+        Optional<IpAddress> fixedIp = associatedVm.ipAddresses().stream().findFirst();
+        if (!fixedIp.isPresent()) {
+            log.warn("Failed to remove floating IP({}) from {}",
+                     floatingIp, associatedVm);
+            return;
+        }
+
+        TrafficSelector.Builder sOutgoingBuilder = DefaultTrafficSelector.builder();
+        TrafficSelector.Builder sIncomingBuilder = DefaultTrafficSelector.builder();
+
+        sOutgoingBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(Long.valueOf(associatedVm.annotations().value(VXLAN_ID)))
+                .matchIPSrc(fixedIp.get().toIpPrefix());
+
+        sIncomingBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(floatingIp.toIpPrefix());
+
+        gatewayService.getGatewayDeviceIds().stream().forEach(deviceId -> {
+            RulePopulatorUtil.removeRule(
+                    flowObjectiveService,
+                    appId,
+                    deviceId,
+                    sOutgoingBuilder.build(),
+                    ForwardingObjective.Flag.VERSATILE,
+                    FLOATING_RULE_PRIORITY);
+
+            RulePopulatorUtil.removeRule(
+                    flowObjectiveService,
+                    appId,
+                    deviceId,
+                    sIncomingBuilder.build(),
+                    ForwardingObjective.Flag.VERSATILE,
+                    FLOATING_RULE_PRIORITY);
+        });
+    }
+
+    private void populateFloatingIpIncomingRules(IpAddress floatingIp, Host associatedVm) {
+        DeviceId cnodeId = associatedVm.location().deviceId();
+        Optional<IpAddress> dataIp = nodeService.dataIp(cnodeId);
+        Optional<IpAddress> fixedIp = associatedVm.ipAddresses().stream().findFirst();
+
+        if (!fixedIp.isPresent() || !dataIp.isPresent()) {
+            log.warn("Failed to associate floating IP({})", floatingIp);
+            return;
+        }
+
+        TrafficSelector selector = DefaultTrafficSelector.builder()
+                .matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(floatingIp.toIpPrefix())
+                .build();
+
+        gatewayService.getGatewayDeviceIds().stream().forEach(gnodeId -> {
+            TrafficTreatment treatment =  DefaultTrafficTreatment.builder()
+                    .setEthSrc(Constants.DEFAULT_GATEWAY_MAC)
+                    .setEthDst(associatedVm.mac())
+                    .setIpDst(associatedVm.ipAddresses().stream().findFirst().get())
+                    .setTunnelId(Long.valueOf(associatedVm.annotations().value(VXLAN_ID)))
+                    .extension(buildExtension(deviceService, cnodeId, dataIp.get().getIp4Address()),
+                               cnodeId)
+                    .setOutput(nodeService.tunnelPort(gnodeId).get())
+                    .build();
+
+            ForwardingObjective fo = DefaultForwardingObjective.builder()
+                    .withSelector(selector)
+                    .withTreatment(treatment)
+                    .withFlag(ForwardingObjective.Flag.VERSATILE)
+                    .withPriority(FLOATING_RULE_PRIORITY)
+                    .fromApp(appId)
+                    .add();
+
+            flowObjectiveService.forward(gnodeId, fo);
+        });
+    }
+
+    private void populateFloatingIpOutgoingRules(IpAddress floatingIp, Host associatedVm) {
+        TrafficSelector selector = DefaultTrafficSelector.builder()
+                .matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(Long.valueOf(associatedVm.annotations().value(VXLAN_ID)))
+                .matchIPSrc(associatedVm.ipAddresses().stream().findFirst().get().toIpPrefix())
+                .build();
+
+        gatewayService.getGatewayDeviceIds().stream().forEach(gnodeId -> {
+            TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+                    .setIpSrc(floatingIp)
+                    .setEthSrc(Constants.DEFAULT_GATEWAY_MAC)
+                    .setEthDst(Constants.DEFAULT_EXTERNAL_ROUTER_MAC)
+                    .setOutput(gatewayService.getUplinkPort(gnodeId))
+                    .build();
+
+            ForwardingObjective fo = DefaultForwardingObjective.builder()
+                    .withSelector(selector)
+                    .withTreatment(treatment)
+                    .withFlag(ForwardingObjective.Flag.VERSATILE)
+                    .withPriority(FLOATING_RULE_PRIORITY)
+                    .fromApp(appId)
+                    .add();
+
+            flowObjectiveService.forward(gnodeId, fo);
+        });
+    }
+
+    private void reloadFloatingIpRules() {
+        floatingIpMap.entrySet().stream().forEach(entry -> {
+            IpAddress floatingIp = entry.getKey();
+            Host associatedVm = entry.getValue().value();
+
+            populateFloatingIpRules(floatingIp, associatedVm);
+            log.debug("Reload floating IP {} mapped to {}",
+                      floatingIp, associatedVm.ipAddresses());
+        });
+    }
+
+    // TODO apply existing floating IPs on service start-up by handling host event
+    // TODO consider the case that port with associated floating IP is attached to a VM
+
+    private class InternalNodeListener implements OpenstackNodeListener {
+
+        @Override
+        public void event(OpenstackNodeEvent event) {
+            OpenstackNode node = event.node();
+
+            switch (event.type()) {
+                case COMPLETE:
+                    if (node.type() == GATEWAY) {
+                        log.info("GATEWAY node {} detected", node.hostname());
+                        GatewayNode gnode = GatewayNode.builder()
+                                .gatewayDeviceId(node.intBridge())
+                                .dataIpAddress(node.dataIp().getIp4Address())
+                                .uplinkIntf(node.externalPortName().get())
+                                .build();
+                        gatewayService.addGatewayNode(gnode);
+                        eventExecutor.execute(OpenstackFloatingIpManager.this::reloadFloatingIpRules);
+                    }
+                    break;
+                case INIT:
+                case DEVICE_CREATED:
+                case INCOMPLETE:
+                default:
+                    break;
+            }
+        }
+    }
+}

@@ -16,322 +16,319 @@
 package org.onosproject.openstacknetworking.routing;
 package org.onosproject.openstacknetworking.routing;
 
 
 import com.google.common.collect.Maps;
 import com.google.common.collect.Maps;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.ICMP;
 import org.onlab.packet.ICMP;
 import org.onlab.packet.IPv4;
 import org.onlab.packet.IPv4;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.Ip4Address;
-import org.onlab.packet.IpAddress;
-import org.onlab.packet.MacAddress;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.Host;
 import org.onosproject.net.Host;
-import org.onosproject.net.Port;
-import org.onosproject.net.PortNumber;
-import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.flow.DefaultTrafficSelector;
 import org.onosproject.net.flow.DefaultTrafficSelector;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
 import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.host.HostService;
 import org.onosproject.net.host.HostService;
 import org.onosproject.net.packet.DefaultOutboundPacket;
 import org.onosproject.net.packet.DefaultOutboundPacket;
+import org.onosproject.net.packet.InboundPacket;
 import org.onosproject.net.packet.OutboundPacket;
 import org.onosproject.net.packet.OutboundPacket;
 import org.onosproject.net.packet.PacketContext;
 import org.onosproject.net.packet.PacketContext;
 import org.onosproject.net.packet.PacketPriority;
 import org.onosproject.net.packet.PacketPriority;
+import org.onosproject.net.packet.PacketProcessor;
 import org.onosproject.net.packet.PacketService;
 import org.onosproject.net.packet.PacketService;
+import org.onosproject.openstackinterface.OpenstackRouter;
 import org.onosproject.openstacknetworking.Constants;
 import org.onosproject.openstacknetworking.Constants;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstacknode.OpenstackNode;
 import org.onosproject.openstacknode.OpenstackNode;
+import org.onosproject.openstacknode.OpenstackNodeEvent;
+import org.onosproject.openstacknode.OpenstackNodeListener;
 import org.onosproject.openstacknode.OpenstackNodeService;
 import org.onosproject.openstacknode.OpenstackNodeService;
 import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
 import java.util.Map;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Optional;
+import java.util.concurrent.ExecutorService;
 
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.openstacknetworking.Constants.*;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.GATEWAY;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.slf4j.LoggerFactory.getLogger;
 
 
 
 
 /**
 /**
- * Handle ICMP packet sent from Openstack Gateway nodes.
+ * Handle ICMP packet sent from OpenStack Gateway nodes.
+ * For a request to any private network gateway IPs, it generates fake reply.
+ * For a request to the external network, it does source NAT with a public IP and
+ * forward the request to the external only if the request instance has external
+ * connection setups.
  */
  */
+@Component(immediate = true)
 public class OpenstackIcmpHandler {
 public class OpenstackIcmpHandler {
     protected final Logger log = getLogger(getClass());
     protected final Logger log = getLogger(getClass());
 
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected CoreService coreService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected PacketService packetService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected HostService hostService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected OpenstackInterfaceService openstackService;
 
 
-    private static final String NETWORK_ROUTER_INTERFACE = "network:router_interface";
-    private static final String PORTNAME = "portName";
-    private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
-    private static final String NETWORK_FLOATING_IP = "network:floatingip";
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ScalableGatewayService gatewayService;
 
 
-    private final PacketService packetService;
-    private final DeviceService deviceService;
-    private final ScalableGatewayService gatewayService;
-    private final HostService hostService;
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected OpenstackNodeService nodeService;
+
+    private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(
+            groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+    private final InternalPacketProcessor packetProcessor = new InternalPacketProcessor();
+    private final InternalNodeListener nodeListener = new InternalNodeListener();
     private final Map<String, Host> icmpInfoMap = Maps.newHashMap();
     private final Map<String, Host> icmpInfoMap = Maps.newHashMap();
-    private final OpenstackInterfaceService openstackService;
-    private final OpenstackNodeService nodeService;
-
-    /**
-     * Default constructor.
-     *
-     * @param packetService             packet service
-     * @param deviceService             device service
-     * @param openstackService          openstackInterface service
-     */
-    OpenstackIcmpHandler(PacketService packetService,
-                         DeviceService deviceService,
-                         HostService hostService,
-                         OpenstackInterfaceService openstackService,
-                         OpenstackNodeService nodeService,
-                         ScalableGatewayService gatewayService
-                         ) {
-        this.packetService = packetService;
-        this.deviceService = deviceService;
-        this.hostService = hostService;
-        this.openstackService = checkNotNull(openstackService);
-        this.nodeService = nodeService;
-        this.gatewayService = gatewayService;
+
+    ApplicationId appId;
+
+    @Activate
+    protected void activate() {
+        appId = coreService.registerApplication(ROUTING_APP_ID);
+        packetService.addProcessor(packetProcessor, PacketProcessor.director(1));
+        nodeService.addListener(nodeListener);
+        requestPacket(appId);
+
+        log.info("Started");
     }
     }
 
 
-    /**
-     * Requests ICMP packet.
-     *
-     * @param appId Application Id
-     */
-    public void requestPacket(ApplicationId appId) {
+    @Deactivate
+    protected void deactivate() {
+        packetService.removeProcessor(packetProcessor);
+        log.info("Stopped");
+    }
+
+    private void requestPacket(ApplicationId appId) {
         TrafficSelector icmpSelector = DefaultTrafficSelector.builder()
         TrafficSelector icmpSelector = DefaultTrafficSelector.builder()
                 .matchEthType(Ethernet.TYPE_IPV4)
                 .matchEthType(Ethernet.TYPE_IPV4)
                 .matchIPProtocol(IPv4.PROTOCOL_ICMP)
                 .matchIPProtocol(IPv4.PROTOCOL_ICMP)
                 .build();
                 .build();
 
 
-       // TODO: Return the correct gateway node
-        Optional<OpenstackNode> gwNode =  nodeService.nodes().stream()
-                .filter(n -> n.type().equals(OpenstackNodeService.NodeType.GATEWAY))
-                .findFirst();
-
-        if (!gwNode.isPresent()) {
-            log.warn("No Gateway is defined.");
-            return;
-        }
-
-        packetService.requestPackets(icmpSelector,
-                PacketPriority.CONTROL,
-                appId,
-                Optional.of(gwNode.get().intBridge()));
+        gatewayService.getGatewayDeviceIds().stream().forEach(gateway -> {
+            packetService.requestPackets(icmpSelector,
+                                         PacketPriority.CONTROL,
+                                         appId,
+                                         Optional.of(gateway));
+            log.debug("Requested ICMP packet on {}", gateway);
+        });
     }
     }
 
 
-    /**
-     * Handles ICMP packet.
-     *
-     * @param context  packet context
-     * @param ethernet ethernet
-     */
-    public void processIcmpPacket(PacketContext context, Ethernet ethernet) {
-        checkNotNull(context, "context can not be null");
-        checkNotNull(ethernet, "ethernet can not be null");
-
+    private void processIcmpPacket(PacketContext context, Ethernet ethernet) {
         IPv4 ipPacket = (IPv4) ethernet.getPayload();
         IPv4 ipPacket = (IPv4) ethernet.getPayload();
-
-        log.debug("icmpEvent called from ip {}, mac {}", Ip4Address.valueOf(ipPacket.getSourceAddress()).toString(),
-                ethernet.getSourceMAC().toString());
+        log.trace("Processing ICMP packet from ip {}, mac {}",
+                  Ip4Address.valueOf(ipPacket.getSourceAddress()),
+                  ethernet.getSourceMAC());
 
 
         ICMP icmp = (ICMP) ipPacket.getPayload();
         ICMP icmp = (ICMP) ipPacket.getPayload();
         short icmpId = getIcmpId(icmp);
         short icmpId = getIcmpId(icmp);
 
 
-        DeviceId deviceId = context.inPacket().receivedFrom().deviceId();
-        PortNumber portNumber = context.inPacket().receivedFrom().port();
-        if (icmp.getIcmpType() == ICMP.TYPE_ECHO_REQUEST) {
-            //TODO: Considers icmp between internal subnets which are belonged to the same router.
-            Optional<Host> host = hostService.getHostsByMac(ethernet.getSourceMAC()).stream().findFirst();
-            if (!host.isPresent()) {
-                log.warn("No host found for MAC {}", ethernet.getSourceMAC());
-                return;
-            }
-
-            IpAddress gatewayIp = IpAddress.valueOf(host.get().annotations().value(Constants.GATEWAY_IP));
-            if (ipPacket.getDestinationAddress() == gatewayIp.getIp4Address().toInt()) {
-                processIcmpPacketSentToGateway(ipPacket, icmp, host.get());
-            } else {
-                Ip4Address pNatIpAddress = pNatIpForPort(host.get());
-                checkNotNull(pNatIpAddress, "pNatIpAddress can not be null");
-
-                sendRequestPacketToExt(ipPacket, icmp, deviceId, pNatIpAddress);
-
+        DeviceId srcDevice = context.inPacket().receivedFrom().deviceId();
+        switch (icmp.getIcmpType()) {
+            case ICMP.TYPE_ECHO_REQUEST:
+                Optional<Host> reqHost = hostService.getHostsByMac(ethernet.getSourceMAC())
+                        .stream().findFirst();
+                if (!reqHost.isPresent()) {
+                    log.warn("No host found for MAC {}", ethernet.getSourceMAC());
+                    return;
+                }
+
+                // TODO Considers icmp between internal subnets belong to the same router.
+                // TODO do we have to support ICMP reply for non-existing gateway?
+                Ip4Address gatewayIp = Ip4Address.valueOf(
+                        reqHost.get().annotations().value(Constants.GATEWAY_IP));
+                if (Objects.equals(ipPacket.getDestinationAddress(), gatewayIp.toInt())) {
+                    processRequestToGateway(ipPacket, reqHost.get());
+                } else {
+                    Optional<Ip4Address> srcNatIp = getSrcNatIp(reqHost.get());
+                    if (!srcNatIp.isPresent()) {
+                        log.trace("VM {} has no external connection", reqHost.get());
+                        return;
+                    }
+
+                    sendRequestToExternal(ipPacket, srcDevice, srcNatIp.get());
+                    String icmpInfoKey = String.valueOf(icmpId)
+                            .concat(String.valueOf(srcNatIp.get().toInt()))
+                            .concat(String.valueOf(ipPacket.getDestinationAddress()));
+                    icmpInfoMap.putIfAbsent(icmpInfoKey, reqHost.get());
+                }
+                break;
+            case ICMP.TYPE_ECHO_REPLY:
                 String icmpInfoKey = String.valueOf(icmpId)
                 String icmpInfoKey = String.valueOf(icmpId)
-                        .concat(String.valueOf(pNatIpAddress.toInt()))
-                        .concat(String.valueOf(ipPacket.getDestinationAddress()));
-                icmpInfoMap.putIfAbsent(icmpInfoKey, host.get());
-            }
-        } else if (icmp.getIcmpType() == ICMP.TYPE_ECHO_REPLY) {
-            String icmpInfoKey = String.valueOf(icmpId)
-                    .concat(String.valueOf(ipPacket.getDestinationAddress()))
-                    .concat(String.valueOf(ipPacket.getSourceAddress()));
-
-            processResponsePacketFromExternalToHost(ipPacket, icmp, icmpInfoMap.get(icmpInfoKey));
-
-            icmpInfoMap.remove(icmpInfoKey);
+                        .concat(String.valueOf(ipPacket.getDestinationAddress()))
+                        .concat(String.valueOf(ipPacket.getSourceAddress()));
+
+                processReplyFromExternal(ipPacket, icmpInfoMap.get(icmpInfoKey));
+                icmpInfoMap.remove(icmpInfoKey);
+                break;
+            default:
+                break;
         }
         }
     }
     }
 
 
-    private void processIcmpPacketSentToExtenal(IPv4 icmpRequestIpv4, ICMP icmpRequest,
-                                                int destAddr, MacAddress destMac,
-                                                DeviceId deviceId, PortNumber portNumber) {
-        icmpRequest.setChecksum((short) 0);
-        icmpRequest.setIcmpType(ICMP.TYPE_ECHO_REPLY).resetChecksum();
-        icmpRequestIpv4.setSourceAddress(icmpRequestIpv4.getDestinationAddress())
-                .setDestinationAddress(destAddr).resetChecksum();
-        icmpRequestIpv4.setPayload(icmpRequest);
-        Ethernet icmpResponseEth = new Ethernet();
-        icmpResponseEth.setEtherType(Ethernet.TYPE_IPV4)
-                // TODO: Get the correct GW MAC
-                .setSourceMACAddress(Constants.GW_EXT_INT_MAC)
-                .setDestinationMACAddress(destMac).setPayload(icmpRequestIpv4);
-        TrafficTreatment treatment = DefaultTrafficTreatment.builder().setOutput(portNumber).build();
-        OutboundPacket packet = new DefaultOutboundPacket(deviceId,
-                treatment, ByteBuffer.wrap(icmpResponseEth.serialize()));
-        packetService.emit(packet);
-    }
-
-    private void processIcmpPacketSentToGateway(IPv4 icmpRequestIpv4, ICMP icmpRequest,
-                                                Host host) {
-        icmpRequest.setChecksum((short) 0);
-        icmpRequest.setIcmpType(ICMP.TYPE_ECHO_REPLY)
-                .resetChecksum();
+    // TODO do we have to handle the request to the fake gateway?
+    private void processRequestToGateway(IPv4 ipPacket, Host reqHost) {
+        ICMP icmpReq = (ICMP) ipPacket.getPayload();
+        icmpReq.setChecksum((short) 0);
+        icmpReq.setIcmpType(ICMP.TYPE_ECHO_REPLY).resetChecksum();
 
 
-        Ip4Address ipAddress = host.ipAddresses().stream().findAny().get().getIp4Address();
-        icmpRequestIpv4.setSourceAddress(icmpRequestIpv4.getDestinationAddress())
-                .setDestinationAddress(ipAddress.toInt())
+        ipPacket.setSourceAddress(ipPacket.getDestinationAddress())
+                .setDestinationAddress(ipPacket.getSourceAddress())
                 .resetChecksum();
                 .resetChecksum();
 
 
-        icmpRequestIpv4.setPayload(icmpRequest);
+        ipPacket.setPayload(icmpReq);
+        Ethernet icmpReply = new Ethernet();
+        icmpReply.setEtherType(Ethernet.TYPE_IPV4)
+                .setSourceMACAddress(Constants.DEFAULT_GATEWAY_MAC)
+                .setDestinationMACAddress(reqHost.mac())
+                .setPayload(icmpReq);
 
 
-        Ethernet icmpResponseEth = new Ethernet();
-
-        icmpResponseEth.setEtherType(Ethernet.TYPE_IPV4)
-                .setSourceMACAddress(Constants.GATEWAY_MAC)
-                .setDestinationMACAddress(host.mac())
-                .setPayload(icmpRequestIpv4);
-
-        sendResponsePacketToHost(icmpResponseEth, host);
+        sendReply(icmpReply, reqHost);
     }
     }
 
 
-    private void sendRequestPacketToExt(IPv4 icmpRequestIpv4, ICMP icmpRequest, DeviceId deviceId,
-                                        Ip4Address pNatIpAddress) {
-        icmpRequest.resetChecksum();
-        icmpRequestIpv4.setSourceAddress(pNatIpAddress.toInt())
-                .resetChecksum();
-        icmpRequestIpv4.setPayload(icmpRequest);
+    private void sendRequestToExternal(IPv4 ipPacket, DeviceId srcDevice, Ip4Address srcNatIp) {
+        ICMP icmpReq = (ICMP) ipPacket.getPayload();
+        icmpReq.resetChecksum();
+        ipPacket.setSourceAddress(srcNatIp.toInt()).resetChecksum();
+        ipPacket.setPayload(icmpReq);
 
 
         Ethernet icmpRequestEth = new Ethernet();
         Ethernet icmpRequestEth = new Ethernet();
-
         icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4)
         icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4)
-                // TODO: Get the correct one - Scalable Gateway ...
-                .setSourceMACAddress(Constants.GW_EXT_INT_MAC)
-                .setDestinationMACAddress(Constants.PHY_ROUTER_MAC)
-                .setPayload(icmpRequestIpv4);
-
-        // TODO: Return the correct gateway node
-        Optional<OpenstackNode> gwNode =  nodeService.nodes().stream()
-                .filter(n -> n.type().equals(OpenstackNodeService.NodeType.GATEWAY))
-                .findFirst();
-
-        if (!gwNode.isPresent()) {
-            log.warn("No Gateway is defined.");
-            return;
-        }
+                .setSourceMACAddress(DEFAULT_GATEWAY_MAC)
+                .setDestinationMACAddress(DEFAULT_EXTERNAL_ROUTER_MAC)
+                .setPayload(ipPacket);
 
 
         TrafficTreatment treatment = DefaultTrafficTreatment.builder()
         TrafficTreatment treatment = DefaultTrafficTreatment.builder()
-                // FIXME: please double check this.
-                .setOutput(getPortForAnnotationPortName(gwNode.get().intBridge(),
-                        // FIXME: please double check this.
-                        org.onosproject.openstacknode.Constants.PATCH_INTG_BRIDGE))
+                .setOutput(gatewayService.getUplinkPort(srcDevice))
                 .build();
                 .build();
 
 
-        OutboundPacket packet = new DefaultOutboundPacket(deviceId,
-                treatment, ByteBuffer.wrap(icmpRequestEth.serialize()));
+        OutboundPacket packet = new DefaultOutboundPacket(
+                srcDevice,
+                treatment,
+                ByteBuffer.wrap(icmpRequestEth.serialize()));
 
 
         packetService.emit(packet);
         packetService.emit(packet);
     }
     }
 
 
-    private void processResponsePacketFromExternalToHost(IPv4 icmpResponseIpv4, ICMP icmpResponse,
-                                                         Host host) {
-        icmpResponse.resetChecksum();
+    private void processReplyFromExternal(IPv4 ipPacket, Host dstHost) {
+        ICMP icmpReply = (ICMP) ipPacket.getPayload();
+        icmpReply.resetChecksum();
 
 
-        Ip4Address ipAddress = host.ipAddresses().stream().findFirst().get().getIp4Address();
-        icmpResponseIpv4.setDestinationAddress(ipAddress.toInt())
+        Ip4Address ipAddress = dstHost.ipAddresses().stream().findFirst().get().getIp4Address();
+        ipPacket.setDestinationAddress(ipAddress.toInt())
                 .resetChecksum();
                 .resetChecksum();
-        icmpResponseIpv4.setPayload(icmpResponse);
+        ipPacket.setPayload(icmpReply);
 
 
         Ethernet icmpResponseEth = new Ethernet();
         Ethernet icmpResponseEth = new Ethernet();
-
         icmpResponseEth.setEtherType(Ethernet.TYPE_IPV4)
         icmpResponseEth.setEtherType(Ethernet.TYPE_IPV4)
-                .setSourceMACAddress(Constants.GATEWAY_MAC)
-                .setDestinationMACAddress(host.mac())
-                .setPayload(icmpResponseIpv4);
+                .setSourceMACAddress(Constants.DEFAULT_GATEWAY_MAC)
+                .setDestinationMACAddress(dstHost.mac())
+                .setPayload(ipPacket);
 
 
-        sendResponsePacketToHost(icmpResponseEth, host);
+        sendReply(icmpResponseEth, dstHost);
     }
     }
 
 
-    private void sendResponsePacketToHost(Ethernet icmpResponseEth, Host host) {
-
+    private void sendReply(Ethernet icmpReply, Host dstHost) {
         TrafficTreatment treatment = DefaultTrafficTreatment.builder()
         TrafficTreatment treatment = DefaultTrafficTreatment.builder()
-                .setOutput(host.location().port())
+                .setOutput(dstHost.location().port())
                 .build();
                 .build();
 
 
-        OutboundPacket packet = new DefaultOutboundPacket(host.location().deviceId(),
-                treatment, ByteBuffer.wrap(icmpResponseEth.serialize()));
+        OutboundPacket packet = new DefaultOutboundPacket(
+                dstHost.location().deviceId(),
+                treatment,
+                ByteBuffer.wrap(icmpReply.serialize()));
 
 
         packetService.emit(packet);
         packetService.emit(packet);
     }
     }
 
 
-    private short getIcmpId(ICMP icmp) {
-        return ByteBuffer.wrap(icmp.serialize(), 4, 2).getShort();
-    }
-
-    private Ip4Address pNatIpForPort(Host host) {
+    private Optional<Ip4Address> getSrcNatIp(Host host) {
+        // TODO cache external gateway IP for each network because
+        // asking Neutron for every ICMP request is a bad idea
+        Optional<OpenstackPort> osPort = openstackService.ports().stream()
+                .filter(port -> port.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+                        Objects.equals(host.annotations().value(NETWORK_ID),
+                                       port.networkId()))
+                .findAny();
+        if (!osPort.isPresent()) {
+            return Optional.empty();
+        }
 
 
-        OpenstackPort openstackPort = openstackService.ports().stream()
-                .filter(p -> p.deviceOwner().equals(NETWORK_ROUTER_INTERFACE) &&
-                        p.networkId().equals(host.annotations().value(Constants.NETWORK_ID)))
-                .findAny().orElse(null);
+        OpenstackRouter osRouter = openstackService.router(osPort.get().deviceId());
+        if (osRouter == null) {
+            return Optional.empty();
+        }
 
 
-        checkNotNull(openstackPort, "openstackPort can not be null");
+        return osRouter.gatewayExternalInfo().externalFixedIps()
+                .values().stream().findAny();
+    }
 
 
-        return openstackService.router(openstackPort.deviceId())
-                .gatewayExternalInfo().externalFixedIps().values()
-                .stream().findAny().orElse(null);
+    private short getIcmpId(ICMP icmp) {
+        return ByteBuffer.wrap(icmp.serialize(), 4, 2).getShort();
     }
     }
 
 
-    private PortNumber getPortForAnnotationPortName(DeviceId deviceId, String match) {
-        Port port = deviceService.getPorts(deviceId).stream()
-                .filter(p -> p.annotations().value(PORTNAME).equals(match))
-                .findAny().orElse(null);
+    private class InternalPacketProcessor implements PacketProcessor {
+
+        @Override
+        public void process(PacketContext context) {
+            if (context.isHandled()) {
+                return;
+            } else if (!gatewayService.getGatewayDeviceIds().contains(
+                    context.inPacket().receivedFrom().deviceId())) {
+                // return if the packet is not from gateway nodes
+                return;
+            }
 
 
-        checkNotNull(port, "port cannot be null");
+            InboundPacket pkt = context.inPacket();
+            Ethernet ethernet = pkt.parsed();
+            if (ethernet == null || ethernet.getEtherType() == Ethernet.TYPE_ARP) {
+                return;
+            }
 
 
-        return port.number();
+            IPv4 iPacket = (IPv4) ethernet.getPayload();
+            if (iPacket.getProtocol() == IPv4.PROTOCOL_ICMP) {
+                    eventExecutor.execute(() -> processIcmpPacket(context, ethernet));
+            }
+        }
     }
     }
 
 
-    private boolean requestToOpenstackRoutingNetwork(int destAddr) {
-        OpenstackPort port = openstackService.ports().stream()
-                .filter(p -> p.deviceOwner().equals(NETWORK_ROUTER_GATEWAY) ||
-                        p.deviceOwner().equals(NETWORK_FLOATING_IP))
-                .filter(p -> p.fixedIps().containsValue(
-                        Ip4Address.valueOf(destAddr)))
-                .findAny().orElse(null);
-        if (port == null) {
-            return false;
+    private class InternalNodeListener implements OpenstackNodeListener {
+
+        @Override
+        public void event(OpenstackNodeEvent event) {
+            OpenstackNode node = event.node();
+
+            switch (event.type()) {
+                case COMPLETE:
+                    if (node.type() == GATEWAY) {
+                        log.info("GATEWAY node {} detected", node.hostname());
+                        eventExecutor.execute(() -> requestPacket(appId));
+                    }
+                    break;
+                case INIT:
+                case DEVICE_CREATED:
+                case INCOMPLETE:
+                default:
+                    break;
+            }
         }
         }
-        return true;
-    }
-    private Map<DeviceId, PortNumber> getExternalInfo() {
-        Map<DeviceId, PortNumber> externalInfoMap = Maps.newHashMap();
-        gatewayService.getGatewayDeviceIds().forEach(deviceId ->
-                externalInfoMap.putIfAbsent(deviceId, gatewayService.getUplinkPort(deviceId)));
-        return externalInfoMap;
     }
     }
 }
 }

@@ -15,109 +15,156 @@
  */
  */
 package org.onosproject.openstacknetworking.routing;
 package org.onosproject.openstacknetworking.routing;
 
 
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.IPv4;
 import org.onlab.packet.IPv4;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.TCP;
 import org.onlab.packet.TCP;
+import org.onlab.packet.TpPort;
 import org.onlab.packet.UDP;
 import org.onlab.packet.UDP;
+import org.onlab.util.KryoNamespace;
+import org.onlab.util.Tools;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.DeviceId;
-import org.onosproject.net.Port;
+import org.onosproject.net.Host;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flowobjective.DefaultForwardingObjective;
+import org.onosproject.net.flowobjective.FlowObjectiveService;
+import org.onosproject.net.flowobjective.ForwardingObjective;
+import org.onosproject.net.host.HostService;
 import org.onosproject.net.packet.DefaultOutboundPacket;
 import org.onosproject.net.packet.DefaultOutboundPacket;
 import org.onosproject.net.packet.InboundPacket;
 import org.onosproject.net.packet.InboundPacket;
 import org.onosproject.net.packet.PacketContext;
 import org.onosproject.net.packet.PacketContext;
+import org.onosproject.net.packet.PacketProcessor;
 import org.onosproject.net.packet.PacketService;
 import org.onosproject.net.packet.PacketService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstackinterface.OpenstackRouter;
 import org.onosproject.openstackinterface.OpenstackRouter;
-import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.openstacknetworking.RulePopulatorUtil;
+import org.onosproject.openstacknode.OpenstackNodeService;
 import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.onosproject.scalablegateway.api.ScalableGatewayService;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
 
 
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.onlab.osgi.DefaultServiceDirectory.getService;
-
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.openstacknetworking.Constants.*;
+import static org.slf4j.LoggerFactory.getLogger;
 
 
 /**
 /**
- * Handle NAT packet processing for Managing Flow Rules In Openstack Nodes.
+ * Handle NAT packet processing for managing flow rules in openstack nodes.
  */
  */
-public class OpenstackPnatHandler implements Runnable {
-
-    volatile PacketContext context;
-    private final Logger log = LoggerFactory.getLogger(getClass());
-
-    private final OpenstackRoutingRulePopulator rulePopulator;
-    private final int portNum;
-    private final OpenstackPort openstackPort;
-    private final Port port;
-
-    private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
-    private static final String EXTERNAL_PORT_NULL = "There is no external port in this deviceId []";
-
-    OpenstackPnatHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context,
-                         int portNum, OpenstackPort openstackPort, Port port) {
-        this.rulePopulator = checkNotNull(rulePopulator);
-        this.context = checkNotNull(context);
-        this.portNum = checkNotNull(portNum);
-        this.openstackPort = checkNotNull(openstackPort);
-        this.port = checkNotNull(port);
-    }
+@Component(immediate = true)
+public class OpenstackPnatHandler {
+    private final Logger log = getLogger(getClass());
 
 
-    @Override
-    public void run() {
-        InboundPacket inboundPacket = context.inPacket();
-        Ethernet ethernet = checkNotNull(inboundPacket.parsed());
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected CoreService coreService;
 
 
-        //TODO: Considers IPV6
-        if (ethernet.getEtherType() != Ethernet.TYPE_IPV4) {
-            log.warn("Now, we just consider IP version 4");
-            return;
-        }
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected PacketService packetService;
 
 
-        OpenstackRouter router = getOpenstackRouter(openstackPort);
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected HostService hostService;
 
 
-        MacAddress externalMac = MacAddress.NONE;
-        MacAddress routerMac = MacAddress.NONE;
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected StorageService storageService;
 
 
-        rulePopulator.populatePnatFlowRules(inboundPacket, openstackPort, portNum,
-                getExternalIp(router), externalMac, routerMac);
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected FlowObjectiveService flowObjectiveService;
 
 
-        packetOut((Ethernet) ethernet.clone(), inboundPacket.receivedFrom().deviceId(), portNum, router);
-    }
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DeviceService deviceService;
 
 
-    private OpenstackRouter getOpenstackRouter(OpenstackPort openstackPort) {
-        OpenstackInterfaceService networkingService = getService(OpenstackInterfaceService.class);
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected OpenstackInterfaceService openstackService;
 
 
-        OpenstackPort port = networkingService.ports()
-                .stream()
-                .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
-                .filter(p -> checkSameSubnet(p, openstackPort))
-                .findAny()
-                .orElse(null);
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected OpenstackNodeService nodeService;
 
 
-        return checkNotNull(networkingService.router(port.deviceId()));
-    }
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ScalableGatewayService gatewayService;
 
 
-    private boolean checkSameSubnet(OpenstackPort p, OpenstackPort openstackPort) {
-        String key1 = checkNotNull(p.fixedIps().keySet().stream().findFirst().orElse(null)).toString();
-        String key2 = checkNotNull(openstackPort.fixedIps().keySet().stream().findFirst().orElse(null)).toString();
-        return key1.equals(key2) ? true : false;
-    }
+    private static final KryoNamespace.Builder NUMBER_SERIALIZER = KryoNamespace.newBuilder()
+            .register(KryoNamespaces.API);
+
+    private static final int PNAT_PORT_EXPIRE_TIME = 1200 * 1000;
+    private static final int TP_PORT_MINIMUM_NUM = 1024;
+    private static final int TP_PORT_MAXIMUM_NUM = 65535;
+
+    private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(
+            groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+    private final InternalPacketProcessor packetProcessor = new InternalPacketProcessor();
+
+    private ConsistentMap<Integer, String> tpPortNumMap;
+    private ApplicationId appId;
+
+    @Activate
+    protected void activate() {
+        appId = coreService.registerApplication(ROUTING_APP_ID);
+        tpPortNumMap = storageService.<Integer, String>consistentMapBuilder()
+                .withSerializer(Serializer.using(NUMBER_SERIALIZER.build()))
+                .withName("openstackrouting-tpportnum")
+                .withApplicationId(appId)
+                .build();
 
 
-    private Ip4Address getExternalIp(OpenstackRouter router) {
-        return router.gatewayExternalInfo().externalFixedIps().values().stream().findAny().orElse(null);
+        packetService.addProcessor(packetProcessor, PacketProcessor.director(1));
+        log.info("Started");
     }
     }
 
 
-    private void packetOut(Ethernet ethernet, DeviceId deviceId, int portNum, OpenstackRouter router) {
-        PacketService packetService = getService(PacketService.class);
+    @Deactivate
+    protected void deactivate() {
+        packetService.removeProcessor(packetProcessor);
+        log.info("Stopped");
+    }
 
 
+    private void processPnatPacket(PacketContext context, Ethernet ethernet) {
         IPv4 iPacket = (IPv4) ethernet.getPayload();
         IPv4 iPacket = (IPv4) ethernet.getPayload();
+        InboundPacket inboundPacket = context.inPacket();
+
+        int srcPort = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
+        OpenstackPort osPort = getOpenstackPort(ethernet.getSourceMAC());
+        if (osPort == null) {
+            return;
+        }
+        Ip4Address externalGatewayIp = getExternalGatewayIp(osPort);
+        if (externalGatewayIp == null) {
+            return;
+        }
+
+        populatePnatFlowRules(context.inPacket(),
+                osPort,
+                TpPort.tpPort(srcPort),
+                externalGatewayIp);
+
+        packetOut((Ethernet) ethernet.clone(),
+                  inboundPacket.receivedFrom().deviceId(),
+                  srcPort,
+                  externalGatewayIp);
+    }
 
 
+    private void packetOut(Ethernet ethernet, DeviceId deviceId, int portNum, Ip4Address externalIp) {
+        IPv4 iPacket = (IPv4) ethernet.getPayload();
         TrafficTreatment.Builder treatment = DefaultTrafficTreatment.builder();
         TrafficTreatment.Builder treatment = DefaultTrafficTreatment.builder();
 
 
         switch (iPacket.getProtocol()) {
         switch (iPacket.getProtocol()) {
@@ -136,26 +183,251 @@ public class OpenstackPnatHandler implements Runnable {
                 iPacket.setPayload(udpPacket);
                 iPacket.setPayload(udpPacket);
                 break;
                 break;
             default:
             default:
-                log.error("Temporally, this method can process UDP and TCP protocol.");
+                log.trace("Temporally, this method can process UDP and TCP protocol.");
                 return;
                 return;
         }
         }
 
 
-        iPacket.setSourceAddress(getExternalIp(router).toString());
+        iPacket.setSourceAddress(externalIp.toString());
         iPacket.resetChecksum();
         iPacket.resetChecksum();
         iPacket.setParent(ethernet);
         iPacket.setParent(ethernet);
         ethernet.setPayload(iPacket);
         ethernet.setPayload(iPacket);
 
 
-        ScalableGatewayService gatewayService = getService(ScalableGatewayService.class);
-        GatewayNode gatewayNode = gatewayService.getGatewayNode(deviceId);
-        if (gatewayNode.getUplinkIntf() == null) {
-            log.error(EXTERNAL_PORT_NULL, deviceId.toString());
+        treatment.setOutput(gatewayService.getUplinkPort(deviceId));
+        ethernet.resetChecksum();
+        packetService.emit(new DefaultOutboundPacket(
+                deviceId,
+                treatment.build(),
+                ByteBuffer.wrap(ethernet.serialize())));
+    }
+
+    private int getPortNum(MacAddress sourceMac, int destinationAddress) {
+        int portNum = findUnusedPortNum();
+        if (portNum == 0) {
+            clearPortNumMap();
+            portNum = findUnusedPortNum();
+        }
+        tpPortNumMap.put(portNum, sourceMac.toString().concat(":").concat(String.valueOf(destinationAddress)));
+        return portNum;
+    }
+
+    private int findUnusedPortNum() {
+        for (int i = TP_PORT_MINIMUM_NUM; i < TP_PORT_MAXIMUM_NUM; i++) {
+            if (!tpPortNumMap.containsKey(i)) {
+                return i;
+            }
+        }
+        return 0;
+    }
+
+    private void clearPortNumMap() {
+        tpPortNumMap.entrySet().forEach(e -> {
+            if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
+                tpPortNumMap.remove(e.getKey());
+            }
+        });
+    }
+
+    // TODO there can be multiple routers connected to a particular openstack port
+    // TODO cache router information
+    private Ip4Address getExternalGatewayIp(OpenstackPort osPort) {
+        Optional<OpenstackPort> routerPort = openstackService.ports().stream()
+                .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
+                .filter(p -> checkSameSubnet(p, osPort))
+                .findAny();
+        if (!routerPort.isPresent()) {
+            log.warn("No router is connected to network {}", osPort.networkId());
+            return null;
+        }
+
+        OpenstackRouter osRouter = openstackService.router(routerPort.get().deviceId());
+        if (osRouter == null) {
+            log.warn("Failed to get OpenStack router {}",
+                     routerPort.get().deviceId());
+            return null;
+        }
+
+        return osRouter.gatewayExternalInfo().externalFixedIps().values()
+                .stream().findAny().orElse(null);
+    }
+
+    private OpenstackPort getOpenstackPort(MacAddress srcMac) {
+        Optional<Host> host = hostService.getHostsByMac(srcMac).stream()
+                .filter(h -> h.annotations().value(PORT_ID) != null)
+                .findAny();
+        if (!host.isPresent()) {
+            log.warn("Failed to find a host with MAC:{}", srcMac);
+            return null;
+        }
+        return openstackService.port(host.get().annotations().value(PORT_ID));
+    }
+
+    private boolean checkSameSubnet(OpenstackPort osPortA, OpenstackPort osPortB) {
+        return osPortA.fixedIps().keySet().stream()
+                .anyMatch(subnetId -> osPortB.fixedIps().keySet().contains(subnetId));
+    }
+
+    private void populatePnatFlowRules(InboundPacket inboundPacket,
+                                      OpenstackPort osPort,
+                                      TpPort patPort,
+                                      Ip4Address externalIp) {
+        long vni = getVni(osPort.networkId());
+        populatePnatIncomingFlowRules(vni, externalIp, patPort, inboundPacket);
+        populatePnatOutgoingFlowRules(vni, externalIp, patPort, inboundPacket);
+    }
+
+    private long getVni(String netId) {
+        // TODO remove this and use host vxlan annotation if applicable
+        return Long.parseLong(openstackService.network(netId).segmentId());
+    }
+
+    private void populatePnatOutgoingFlowRules(long vni, Ip4Address externalIp, TpPort patPort,
+                                               InboundPacket inboundPacket) {
+        IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
+
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPProtocol(iPacket.getProtocol())
+                .matchTunnelId(vni)
+                .matchIPSrc(IpPrefix.valueOf(iPacket.getSourceAddress(), 32))
+                .matchIPDst(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
+
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+        switch (iPacket.getProtocol()) {
+            case IPv4.PROTOCOL_TCP:
+                TCP tcpPacket = (TCP) iPacket.getPayload();
+                sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getSourcePort()))
+                        .matchTcpDst(TpPort.tpPort(tcpPacket.getDestinationPort()));
+                tBuilder.setTcpSrc(patPort)
+                        .setEthDst(DEFAULT_EXTERNAL_ROUTER_MAC);
+                break;
+            case IPv4.PROTOCOL_UDP:
+                UDP udpPacket = (UDP) iPacket.getPayload();
+                sBuilder.matchUdpSrc(TpPort.tpPort(udpPacket.getSourcePort()))
+                        .matchUdpDst(TpPort.tpPort(udpPacket.getDestinationPort()));
+                tBuilder.setUdpSrc(patPort)
+                        .setEthDst(DEFAULT_EXTERNAL_ROUTER_MAC);
+
+                break;
+            default:
+                log.debug("Unsupported IPv4 protocol {}");
+                break;
+        }
+
+        tBuilder.setIpSrc(externalIp);
+        gatewayService.getGatewayNodes().stream().forEach(gateway -> {
+            TrafficTreatment.Builder tmpBuilder = tBuilder;
+            tmpBuilder.setOutput(gatewayService.getUplinkPort(gateway.getGatewayDeviceId()));
+            ForwardingObjective fo = DefaultForwardingObjective.builder()
+                    .withSelector(sBuilder.build())
+                    .withTreatment(tmpBuilder.build())
+                    .withFlag(ForwardingObjective.Flag.VERSATILE)
+                    .withPriority(PNAT_RULE_PRIORITY)
+                    .makeTemporary(PNAT_TIMEOUT)
+                    .fromApp(appId)
+                    .add();
+
+            flowObjectiveService.forward(gateway.getGatewayDeviceId(), fo);
+        });
+    }
+
+    private void populatePnatIncomingFlowRules(long vni, Ip4Address externalIp, TpPort patPort,
+                                               InboundPacket inboundPacket) {
+        IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
+        IpAddress internalIp = IpAddress.valueOf(iPacket.getSourceAddress());
+
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPProtocol(iPacket.getProtocol())
+                .matchIPDst(IpPrefix.valueOf(externalIp, 32))
+                .matchIPSrc(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
+
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+        tBuilder.setTunnelId(vni)
+                .setEthDst(inboundPacket.parsed().getSourceMAC())
+                .setIpDst(internalIp);
+
+        switch (iPacket.getProtocol()) {
+            case IPv4.PROTOCOL_TCP:
+                TCP tcpPacket = (TCP) iPacket.getPayload();
+                sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getDestinationPort()))
+                        .matchTcpDst(patPort);
+                tBuilder.setTcpDst(TpPort.tpPort(tcpPacket.getSourcePort()));
+                break;
+            case IPv4.PROTOCOL_UDP:
+                UDP udpPacket = (UDP) iPacket.getPayload();
+                sBuilder.matchUdpSrc(TpPort.tpPort(udpPacket.getDestinationPort()))
+                        .matchUdpDst(patPort);
+                tBuilder.setUdpDst(TpPort.tpPort(udpPacket.getSourcePort()));
+                break;
+            default:
+                break;
+        }
+
+        Optional<Host> srcVm = Tools.stream(hostService.getHostsByIp(internalIp))
+                .filter(host -> Objects.equals(
+                        host.annotations().value(VXLAN_ID),
+                        String.valueOf(vni)))
+                .findFirst();
+        if (!srcVm.isPresent()) {
+            log.warn("Failed to find source VM with IP {}", internalIp);
             return;
             return;
         }
         }
-        treatment.setOutput(gatewayService.getUplinkPort(deviceId));
 
 
-        ethernet.resetChecksum();
+        gatewayService.getGatewayDeviceIds().stream().forEach(deviceId -> {
+            DeviceId srcDeviceId = srcVm.get().location().deviceId();
+            TrafficTreatment.Builder tmpBuilder = tBuilder;
+            tmpBuilder.extension(RulePopulatorUtil.buildExtension(
+                    deviceService,
+                    deviceId,
+                    nodeService.dataIp(srcDeviceId).get().getIp4Address()), deviceId)
+                    .setOutput(nodeService.tunnelPort(deviceId).get());
 
 
-        packetService.emit(new DefaultOutboundPacket(deviceId, treatment.build(),
-                ByteBuffer.wrap(ethernet.serialize())));
+            ForwardingObjective fo = DefaultForwardingObjective.builder()
+                    .withSelector(sBuilder.build())
+                    .withTreatment(tmpBuilder.build())
+                    .withFlag(ForwardingObjective.Flag.VERSATILE)
+                    .withPriority(PNAT_RULE_PRIORITY)
+                    .makeTemporary(PNAT_TIMEOUT)
+                    .fromApp(appId)
+                    .add();
+
+            flowObjectiveService.forward(deviceId, fo);
+        });
+    }
+
+    private class InternalPacketProcessor implements PacketProcessor {
+
+        @Override
+        public void process(PacketContext context) {
+            if (context.isHandled()) {
+                return;
+            } else if (!gatewayService.getGatewayDeviceIds().contains(
+                    context.inPacket().receivedFrom().deviceId())) {
+                // return if the packet is not from gateway nodes
+                return;
+            }
+
+            InboundPacket pkt = context.inPacket();
+            Ethernet ethernet = pkt.parsed();
+            if (ethernet == null || ethernet.getEtherType() == Ethernet.TYPE_ARP) {
+                return;
+            }
+
+            IPv4 iPacket = (IPv4) ethernet.getPayload();
+            switch (iPacket.getProtocol()) {
+                case IPv4.PROTOCOL_ICMP:
+                    break;
+                case IPv4.PROTOCOL_UDP:
+                    UDP udpPacket = (UDP) iPacket.getPayload();
+                    if (udpPacket.getDestinationPort() == UDP.DHCP_SERVER_PORT &&
+                            udpPacket.getSourcePort() == UDP.DHCP_CLIENT_PORT) {
+                        // don't process DHCP
+                        break;
+                    }
+                default:
+                    eventExecutor.execute(() -> processPnatPacket(context, ethernet));
+                    break;
+            }
+        }
     }
     }
 }
 }
\ No newline at end of file

@@ -15,99 +15,74 @@
  */
  */
 package org.onosproject.openstacknetworking.routing;
 package org.onosproject.openstacknetworking.routing;
 
 
-import com.google.common.collect.Lists;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.packet.ARP;
 import org.onlab.packet.ARP;
-import org.onlab.packet.EthType;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.MacAddress;
-import org.onosproject.core.ApplicationId;
-import org.onosproject.net.DeviceId;
-import org.onosproject.net.flow.DefaultTrafficSelector;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
-import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.packet.DefaultOutboundPacket;
 import org.onosproject.net.packet.DefaultOutboundPacket;
+import org.onosproject.net.packet.InboundPacket;
 import org.onosproject.net.packet.PacketContext;
 import org.onosproject.net.packet.PacketContext;
-import org.onosproject.net.packet.PacketPriority;
+import org.onosproject.net.packet.PacketProcessor;
 import org.onosproject.net.packet.PacketService;
 import org.onosproject.net.packet.PacketService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.onosproject.openstacknetworking.Constants;
 import org.onosproject.openstacknetworking.Constants;
-import org.onosproject.openstacknode.OpenstackNodeService;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Optional;
+import java.util.concurrent.ExecutorService;
 
 
-import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.openstacknetworking.Constants.DEVICE_OWNER_FLOATING_IP;
+import static org.onosproject.openstacknetworking.Constants.DEVICE_OWNER_ROUTER_GATEWAY;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.slf4j.LoggerFactory.getLogger;
 
 
 /**
 /**
- * Handle ARP packet sent from Openstack Gateway nodes.
+ * Handle ARP, ICMP and NAT packets from gateway nodes.
  */
  */
+@Component(immediate = true)
 public class OpenstackRoutingArpHandler {
 public class OpenstackRoutingArpHandler {
-    protected final Logger log = getLogger(getClass());
-
-    private final PacketService packetService;
-    private final OpenstackInterfaceService openstackService;
-    private final ScalableGatewayService gatewayService;
-    private final OpenstackNodeService nodeService;
-    private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
-    private static final String NETWORK_FLOATING_IP = "network:floatingip";
-
-    /**
-     * Default constructor.
-     *
-     * @param packetService packet service
-     * @param openstackService openstackInterface service
-     * @param gatewayService gateway service
-     * @param nodeService openstackNodeService
-     */
-    OpenstackRoutingArpHandler(PacketService packetService, OpenstackInterfaceService openstackService,
-                               OpenstackNodeService nodeService, ScalableGatewayService gatewayService) {
-        this.packetService = packetService;
-        this.openstackService = checkNotNull(openstackService);
-        this.nodeService = nodeService;
-        this.gatewayService = gatewayService;
-    }
+    private final Logger log = getLogger(getClass());
 
 
-    /**
-     * Requests ARP packet to GatewayNode.
-     *
-     * @param appId application id
-     */
-    public void requestPacket(ApplicationId appId) {
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected PacketService packetService;
 
 
-        TrafficSelector arpSelector = DefaultTrafficSelector.builder()
-                .matchEthType(EthType.EtherType.ARP.ethType().toShort())
-                .build();
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected OpenstackInterfaceService openstackService;
 
 
-        getExternalInfo().forEach(deviceId ->
-                packetService.requestPackets(arpSelector,
-                        PacketPriority.CONTROL,
-                        appId,
-                        Optional.of(deviceId))
-        );
-    }
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ScalableGatewayService gatewayService;
 
 
-    /**
-     * Handles ARP packet.
-     *
-     * @param context packet context
-     * @param ethernet ethernet
-     */
-    public void processArpPacketFromRouter(PacketContext context, Ethernet ethernet) {
-        checkNotNull(context, "context can not be null");
-        checkNotNull(ethernet, "ethernet can not be null");
+    private final ExecutorService executorService =
+            newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "packet-event", log));
 
 
+    private final InternalPacketProcessor packetProcessor = new InternalPacketProcessor();
 
 
-        ARP arp = (ARP) ethernet.getPayload();
+    @Activate
+    protected void activate() {
+        packetService.addProcessor(packetProcessor, PacketProcessor.director(1));
+        log.info("Started");
+    }
 
 
-        log.debug("arpEvent called from {} to {}",
+    @Deactivate
+    protected void deactivate() {
+        packetService.removeProcessor(packetProcessor);
+        log.info("Stopped");
+    }
+
+    private void processArpPacket(PacketContext context, Ethernet ethernet) {
+        ARP arp = (ARP) ethernet.getPayload();
+        log.trace("arpEvent called from {} to {}",
                 Ip4Address.valueOf(arp.getSenderProtocolAddress()).toString(),
                 Ip4Address.valueOf(arp.getSenderProtocolAddress()).toString(),
                 Ip4Address.valueOf(arp.getTargetProtocolAddress()).toString());
                 Ip4Address.valueOf(arp.getTargetProtocolAddress()).toString());
 
 
@@ -116,13 +91,11 @@ public class OpenstackRoutingArpHandler {
         }
         }
 
 
         IpAddress targetIp = Ip4Address.valueOf(arp.getTargetProtocolAddress());
         IpAddress targetIp = Ip4Address.valueOf(arp.getTargetProtocolAddress());
-
         if (getTargetMacForTargetIp(targetIp.getIp4Address()) == MacAddress.NONE) {
         if (getTargetMacForTargetIp(targetIp.getIp4Address()) == MacAddress.NONE) {
-                return;
+            return;
         }
         }
-        // FIXME: Set the correct gateway
-        MacAddress targetMac = Constants.GW_EXT_INT_MAC;
 
 
+        MacAddress targetMac = Constants.DEFAULT_EXTERNAL_ROUTER_MAC;
         Ethernet ethReply = ARP.buildArpReply(targetIp.getIp4Address(),
         Ethernet ethReply = ARP.buildArpReply(targetIp.getIp4Address(),
                 targetMac, ethernet);
                 targetMac, ethernet);
 
 
@@ -136,22 +109,35 @@ public class OpenstackRoutingArpHandler {
                 ByteBuffer.wrap(ethReply.serialize())));
                 ByteBuffer.wrap(ethReply.serialize())));
     }
     }
 
 
+    private class InternalPacketProcessor implements PacketProcessor {
+
+        @Override
+        public void process(PacketContext context) {
+            if (context.isHandled()) {
+                return;
+            } else if (!gatewayService.getGatewayDeviceIds().contains(
+                    context.inPacket().receivedFrom().deviceId())) {
+                // return if the packet is not from gateway nodes
+                return;
+            }
+
+            InboundPacket pkt = context.inPacket();
+            Ethernet ethernet = pkt.parsed();
+            if (ethernet != null &&
+                    ethernet.getEtherType() == Ethernet.TYPE_ARP) {
+                executorService.execute(() -> processArpPacket(context, ethernet));
+            }
+        }
+    }
+
+    // TODO make a cache for the MAC, not a good idea to REST call every time it gets ARP request
     private MacAddress getTargetMacForTargetIp(Ip4Address targetIp) {
     private MacAddress getTargetMacForTargetIp(Ip4Address targetIp) {
         OpenstackPort port = openstackService.ports().stream()
         OpenstackPort port = openstackService.ports().stream()
-                .filter(p -> p.deviceOwner().equals(NETWORK_ROUTER_GATEWAY) ||
-                             p.deviceOwner().equals(NETWORK_FLOATING_IP))
+                .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_GATEWAY) ||
+                        p.deviceOwner().equals(DEVICE_OWNER_FLOATING_IP))
                 .filter(p -> p.fixedIps().containsValue(targetIp.getIp4Address()))
                 .filter(p -> p.fixedIps().containsValue(targetIp.getIp4Address()))
                 .findAny().orElse(null);
                 .findAny().orElse(null);
 
 
-        if (port == null) {
-            return MacAddress.NONE;
-        }
-        return port.macAddress();
-    }
-
-    private List<DeviceId> getExternalInfo() {
-        List<DeviceId> externalInfoList = Lists.newArrayList();
-        gatewayService.getGatewayDeviceIds().forEach(externalInfoList::add);
-        return externalInfoList;
+        return port == null ? MacAddress.NONE : port.macAddress();
     }
     }
 }
 }

@@ -15,8 +15,7 @@
  */
  */
 package org.onosproject.openstacknetworking.routing;
 package org.onosproject.openstacknetworking.routing;
 
 
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.ImmutableSet;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -24,82 +23,62 @@ import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.Service;
 import org.apache.felix.scr.annotations.Service;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ethernet;
-import org.onlab.packet.IPv4;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.Ip4Address;
-import org.onlab.packet.IpAddress;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.MacAddress;
-import org.onlab.packet.UDP;
-import org.onlab.util.KryoNamespace;
+import org.onlab.util.Tools;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.ApplicationId;
-import org.onosproject.core.CoreService;
-import org.onosproject.mastership.MastershipService;
-import org.onosproject.net.DefaultAnnotations;
+import org.onosproject.core.GroupId;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.Host;
 import org.onosproject.net.Host;
-import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
 import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.device.DeviceService;
-import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flowobjective.DefaultForwardingObjective;
 import org.onosproject.net.flowobjective.FlowObjectiveService;
 import org.onosproject.net.flowobjective.FlowObjectiveService;
-import org.onosproject.net.host.DefaultHostDescription;
-import org.onosproject.net.host.HostDescription;
-import org.onosproject.net.host.HostEvent;
-import org.onosproject.net.host.HostListener;
-import org.onosproject.net.host.HostProvider;
-import org.onosproject.net.host.HostProviderRegistry;
-import org.onosproject.net.host.HostProviderService;
-import org.onosproject.net.host.HostService;
-import org.onosproject.net.packet.InboundPacket;
-import org.onosproject.net.packet.PacketContext;
-import org.onosproject.net.packet.PacketProcessor;
-import org.onosproject.net.packet.PacketService;
-import org.onosproject.net.provider.AbstractProvider;
-import org.onosproject.net.provider.ProviderId;
-import org.onosproject.openstackinterface.OpenstackFloatingIP;
+import org.onosproject.net.flowobjective.ForwardingObjective;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
+import org.onosproject.openstackinterface.OpenstackNetwork;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstackinterface.OpenstackRouter;
 import org.onosproject.openstackinterface.OpenstackRouter;
 import org.onosproject.openstackinterface.OpenstackRouterInterface;
 import org.onosproject.openstackinterface.OpenstackRouterInterface;
-import org.onosproject.openstacknetworking.OpenstackRoutingService;
-import org.onosproject.scalablegateway.api.ScalableGatewayService;
-import org.onosproject.openstacknetworking.routing.OpenstackFloatingIPHandler.Action;
+import org.onosproject.openstackinterface.OpenstackSubnet;
+import org.onosproject.openstacknetworking.AbstractVmHandler;
 import org.onosproject.openstacknetworking.Constants;
 import org.onosproject.openstacknetworking.Constants;
+import org.onosproject.openstacknetworking.OpenstackRoutingService;
+import org.onosproject.openstacknetworking.RulePopulatorUtil;
 import org.onosproject.openstacknode.OpenstackNode;
 import org.onosproject.openstacknode.OpenstackNode;
 import org.onosproject.openstacknode.OpenstackNodeEvent;
 import org.onosproject.openstacknode.OpenstackNodeEvent;
 import org.onosproject.openstacknode.OpenstackNodeListener;
 import org.onosproject.openstacknode.OpenstackNodeListener;
 import org.onosproject.openstacknode.OpenstackNodeService;
 import org.onosproject.openstacknode.OpenstackNodeService;
-import org.onosproject.store.serializers.KryoNamespaces;
-import org.onosproject.store.service.ConsistentMap;
-import org.onosproject.store.service.Serializer;
-import org.onosproject.store.service.StorageService;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;
 
 
-import java.util.Collection;
-import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Optional;
 import java.util.Set;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.stream.Collectors;
 import java.util.stream.Collectors;
 
 
-import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
 import static org.onlab.util.Tools.groupedThreads;
 import static org.onlab.util.Tools.groupedThreads;
-import static org.onosproject.net.AnnotationKeys.PORT_NAME;
+import static org.onosproject.openstacknetworking.Constants.*;
+import static org.onosproject.openstacknetworking.RulePopulatorUtil.buildExtension;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.COMPUTE;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.GATEWAY;
 
 
 @Component(immediate = true)
 @Component(immediate = true)
 @Service
 @Service
-/**
- * Populates flow rules about L3 functionality for VMs in Openstack.
- */
-public class OpenstackRoutingManager implements OpenstackRoutingService {
+public class OpenstackRoutingManager extends AbstractVmHandler implements OpenstackRoutingService {
 
 
     private final Logger log = LoggerFactory.getLogger(getClass());
     private final Logger log = LoggerFactory.getLogger(getClass());
 
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected CoreService coreService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected PacketService packetService;
+    protected FlowObjectiveService flowObjectiveService;
 
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected DeviceService deviceService;
     protected DeviceService deviceService;
@@ -108,620 +87,392 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
     protected OpenstackInterfaceService openstackService;
     protected OpenstackInterfaceService openstackService;
 
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected FlowObjectiveService flowObjectiveService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected DriverService driverService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected StorageService storageService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected HostService hostService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected HostProviderRegistry hostProviderRegistry;
+    protected OpenstackNodeService nodeService;
 
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected ScalableGatewayService gatewayService;
     protected ScalableGatewayService gatewayService;
 
 
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected OpenstackNodeService nodeService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected MastershipService mastershipService;
+    private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(
+            groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+    private final InternalNodeListener nodeListener = new InternalNodeListener();
 
 
     private ApplicationId appId;
     private ApplicationId appId;
-    private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
-    private ConsistentMap<String, OpenstackFloatingIP> floatingIpMap; // Map<FloatingIp`s Id, FloatingIp object>
-    // Map<RouterInterface`s portId, Corresponded port`s network id>
-    private ConsistentMap<String, String> routerInterfaceMap;
-    private static final ProviderId PID = new ProviderId("of", "org.onosproject.openstackroutering", true);
-    private static final String APP_ID = "org.onosproject.openstackrouting";
-    private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
-    private static final String FLOATING_IP_MAP_NAME = "openstackrouting-floatingip";
-    private static final String TP_PORT_MAP_NAME = "openstackrouting-tpportnum";
-    private static final String ROUTER_INTERFACE_MAP_NAME = "openstackrouting-routerinterface";
-    private static final String COLON = ":";
-    private static final int PNAT_PORT_EXPIRE_TIME = 1200 * 1000;
-    private static final int TP_PORT_MINIMUM_NUM = 1024;
-    private static final int TP_PORT_MAXIMUM_NUM = 65535;
-
-    private static final KryoNamespace.Builder FLOATING_IP_SERIALIZER = KryoNamespace.newBuilder()
-            .register(KryoNamespaces.API)
-            .register(OpenstackFloatingIP.FloatingIpStatus.class)
-            .register(OpenstackFloatingIP.class);
-
-    private static final KryoNamespace.Builder NUMBER_SERIALIZER = KryoNamespace.newBuilder()
-            .register(KryoNamespaces.API);
-
-    private static final KryoNamespace.Builder ROUTER_INTERFACE_SERIALIZER = KryoNamespace.newBuilder()
-            .register(KryoNamespaces.API);
-
-    private InternalPacketProcessor internalPacketProcessor = new InternalPacketProcessor();
-    private InternalHostListener internalHostListener = new InternalHostListener();
-    private InternalOpenstackNodeListener internalNodeListener = new InternalOpenstackNodeListener();
-    private ExecutorService l3EventExecutorService =
-            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "L3-event"));
-    private ExecutorService icmpEventExecutorService =
-            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "icmp-event"));
-    private ExecutorService arpEventExecutorService =
-            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "arp-event"));
-    private OpenstackIcmpHandler openstackIcmpHandler;
-    private OpenstackRoutingArpHandler openstackArpHandler;
-    private OpenstackRoutingRulePopulator rulePopulator;
-
-    private HostProviderService hostProviderService;
-    private final HostProvider hostProvider = new InternalHostProvider();
 
 
     @Activate
     @Activate
     protected void activate() {
     protected void activate() {
-        appId = coreService.registerApplication(APP_ID);
-        hostService.addListener(internalHostListener);
-        nodeService.addListener(internalNodeListener);
-        hostProviderService = hostProviderRegistry.register(hostProvider);
-
-        floatingIpMap = storageService.<String, OpenstackFloatingIP>consistentMapBuilder()
-                .withSerializer(Serializer.using(FLOATING_IP_SERIALIZER.build()))
-                .withName(FLOATING_IP_MAP_NAME)
-                .withApplicationId(appId)
-                .build();
-        tpPortNumMap = storageService.<Integer, String>consistentMapBuilder()
-                .withSerializer(Serializer.using(NUMBER_SERIALIZER.build()))
-                .withName(TP_PORT_MAP_NAME)
-                .withApplicationId(appId)
-                .build();
-        routerInterfaceMap = storageService.<String, String>consistentMapBuilder()
-                .withSerializer(Serializer.using(ROUTER_INTERFACE_SERIALIZER.build()))
-                .withName(ROUTER_INTERFACE_MAP_NAME)
-                .withApplicationId(appId)
-                .build();
-
-        log.info("started");
+        super.activate();
+        appId = coreService.registerApplication(ROUTING_APP_ID);
+        nodeService.addListener(nodeListener);
     }
     }
 
 
     @Deactivate
     @Deactivate
     protected void deactivate() {
     protected void deactivate() {
-        packetService.removeProcessor(internalPacketProcessor);
-        hostService.removeListener(internalHostListener);
-        nodeService.removeListener(internalNodeListener);
-
-        l3EventExecutorService.shutdown();
-        icmpEventExecutorService.shutdown();
-        arpEventExecutorService.shutdown();
-
-        floatingIpMap.clear();
-        tpPortNumMap.clear();
-        routerInterfaceMap.clear();
-
+        nodeService.removeListener(nodeListener);
         log.info("stopped");
         log.info("stopped");
     }
     }
 
 
-
     @Override
     @Override
-    public void createFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
-        floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
+    public void createRouter(OpenstackRouter osRouter) {
     }
     }
 
 
     @Override
     @Override
-    public void updateFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
-        if (!floatingIpMap.containsKey(openstackFloatingIp.id())) {
-            log.warn("There`s no information about {} in FloatingIpMap", openstackFloatingIp.id());
-            return;
-        }
-        if (openstackFloatingIp.portId() == null || openstackFloatingIp.portId().equals("null")) {
-            OpenstackFloatingIP floatingIp = floatingIpMap.get(openstackFloatingIp.id()).value();
-            // XXX When the VM has been removed, host information has been removed or not ???
-            Optional<Host> host = hostService.getHostsByIp(openstackFloatingIp.fixedIpAddress().getIp4Address())
-                    .stream()
-                    .findFirst();
-            if (!host.isPresent()) {
-                log.warn("No Host info with the VM IP the Floating IP address {} is found",
-                        openstackFloatingIp.floatingIpAddress());
-                return;
-            }
-            l3EventExecutorService.execute(
-                    new OpenstackFloatingIPHandler(rulePopulator, floatingIp, Action.DISSASSOCIATE, host.get()));
-            floatingIpMap.replace(floatingIp.id(), openstackFloatingIp);
-            registerFloatingIpToHostService(openstackFloatingIp, Action.DISSASSOCIATE);
+    public void updateRouter(OpenstackRouter osRouter) {
+        if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+            openstackService.ports().stream()
+                    .filter(osPort -> osPort.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+                            osPort.deviceId().equals(osRouter.id()))
+                    .forEach(osPort -> setExternalConnection(osRouter, osPort.networkId()));
+
+            log.info("Connected external gateway {} to router {}",
+                     osRouter.gatewayExternalInfo().externalFixedIps(),
+                     osRouter.name());
         } else {
         } else {
-            floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
-            l3EventExecutorService.execute(
-                    new OpenstackFloatingIPHandler(rulePopulator, openstackFloatingIp, Action.ASSOCIATE, null));
-            registerFloatingIpToHostService(openstackFloatingIp, Action.ASSOCIATE);
-        }
-    }
+            openstackService.ports().stream()
+                    .filter(osPort -> osPort.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+                            osPort.deviceId().equals(osRouter.id()))
+                    .forEach(osPort -> unsetExternalConnection(osRouter, osPort.networkId()));
 
 
-    @Override
-    public void deleteFloatingIP(String id) {
-        floatingIpMap.remove(id);
+            log.info("Disconnected external gateway from router {}",
+                     osRouter.name());
+        }
     }
     }
 
 
     @Override
     @Override
-    public void createRouter(OpenstackRouter openstackRouter) {
+    public void removeRouter(String osRouterId) {
+        // TODO implement this
     }
     }
 
 
     @Override
     @Override
-    public void updateRouter(OpenstackRouter openstackRouter) {
-        if (openstackRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
-            checkExternalConnection(openstackRouter, getOpenstackRouterInterface(openstackRouter));
-        } else {
-            unsetExternalConnection();
+    public void addRouterInterface(OpenstackRouterInterface routerIface) {
+        OpenstackRouter osRouter = openstackRouter(routerIface.id());
+        OpenstackPort osPort = openstackService.port(routerIface.portId());
+        if (osRouter == null || osPort == null) {
+            log.warn("Failed to add router interface {}", routerIface);
+            return;
         }
         }
-    }
-
-    private void unsetExternalConnection() {
-        Collection<OpenstackRouter> internalRouters = getExternalRouter(false);
-        internalRouters.forEach(r ->
-                getOpenstackRouterInterface(r).forEach(i -> rulePopulator.removeExternalRules(i)));
-    }
 
 
-    private Collection<OpenstackRouter> getExternalRouter(boolean externalConnection) {
-        List<OpenstackRouter> routers;
-        if (externalConnection) {
-            routers = openstackService.routers()
-                    .stream()
-                    .filter(r -> (r.gatewayExternalInfo().externalFixedIps().size() > 0))
-                    .collect(Collectors.toList());
-        } else {
-            routers = openstackService.routers()
-                    .stream()
-                    .filter(r -> (r.gatewayExternalInfo().externalFixedIps().size() == 0))
-                    .collect(Collectors.toList());
+        setRoutes(osRouter, Optional.empty());
+        if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+            setExternalConnection(osRouter, osPort.networkId());
         }
         }
-        return routers;
-    }
-
-    @Override
-    public void deleteRouter(String id) {
-        //TODO : In now, there`s nothing to do for deleteRouter process. It is reserved.
+        log.info("Connected {} to router {}", osPort.fixedIps(), osRouter.name());
     }
     }
 
 
     @Override
     @Override
-    public void updateRouterInterface(OpenstackRouterInterface routerInterface) {
-        List<OpenstackRouterInterface> routerInterfaces = Lists.newArrayList();
-        routerInterfaces.add(routerInterface);
-        checkExternalConnection(getOpenstackRouter(routerInterface.id()), routerInterfaces);
-        setL3Connection(getOpenstackRouter(routerInterface.id()), null);
-        routerInterfaceMap.put(routerInterface.portId(), openstackService.port(routerInterface.portId()).networkId());
-    }
-
-    /**
-     * Set flow rules for traffic between two different subnets when more than one subnets
-     * connected to a router.
-     *
-     * @param openstackRouter OpenstackRouter Info
-     * @param openstackPort OpenstackPort Info
-     */
-    private void setL3Connection(OpenstackRouter openstackRouter, OpenstackPort openstackPort) {
-        Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(openstackRouter);
-
-        if (interfaceList.size() < 2) {
+    public void removeRouterInterface(OpenstackRouterInterface routerIface) {
+        OpenstackRouter osRouter = openstackService.router(routerIface.id());
+        if (osRouter == null) {
+            log.warn("Failed to remove router interface {}", routerIface);
             return;
             return;
         }
         }
-        if (openstackPort == null) {
-            interfaceList.forEach(i -> {
-                OpenstackPort interfacePort = openstackService.port(i.portId());
-                openstackService.ports()
-                        .stream()
-                        .filter(p -> p.networkId().equals(interfacePort.networkId())
-                                && !p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
-                        .forEach(p -> rulePopulator.populateL3Rules(p,
-                                    getL3ConnectionList(p.networkId(), interfaceList)));
-
-            });
-        } else {
-            rulePopulator.populateL3Rules(openstackPort, getL3ConnectionList(openstackPort.networkId(), interfaceList));
-        }
 
 
-    }
+        OpenstackSubnet osSubnet = openstackService.subnet(routerIface.subnetId());
+        OpenstackNetwork osNet = openstackService.network(osSubnet.networkId());
 
 
-    private List<OpenstackRouterInterface> getL3ConnectionList(String networkId,
-                                                               Collection<OpenstackRouterInterface> interfaceList) {
-        List<OpenstackRouterInterface> targetList = Lists.newArrayList();
-        interfaceList.forEach(i -> {
-            OpenstackPort port = openstackService.port(i.portId());
-            if (!port.networkId().equals(networkId)) {
-                targetList.add(i);
-            }
-        });
-        return targetList;
-    }
-
-    @Override
-    public void removeRouterInterface(OpenstackRouterInterface routerInterface) {
-        OpenstackRouter router = openstackService.router(routerInterface.id());
-        Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(router);
-        if (interfaceList.size() == 1) {
-            List<OpenstackRouterInterface> newList = Lists.newArrayList();
-            newList.add(routerInterface);
-            interfaceList.forEach(i -> removeL3RulesForRouterInterface(i, router, newList));
+        unsetRoutes(osRouter, osNet);
+        if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+            unsetExternalConnection(osRouter, osNet.id());
         }
         }
-        removeL3RulesForRouterInterface(routerInterface, router, null);
-        rulePopulator.removeExternalRules(routerInterface);
-        routerInterfaceMap.remove(routerInterface.portId());
+        log.info("Disconnected {} from router {}", osSubnet.cidr(), osRouter.name());
     }
     }
 
 
-    private void removeL3RulesForRouterInterface(OpenstackRouterInterface routerInterface, OpenstackRouter router,
-                                                 List<OpenstackRouterInterface> newList) {
-        if (!routerInterfaceMap.containsKey(routerInterface.portId())) {
-            log.warn("No router interface information found for {}", routerInterface.portId());
+    private void setExternalConnection(OpenstackRouter osRouter, String osNetId) {
+        if (!osRouter.gatewayExternalInfo().isEnablePnat()) {
+            log.debug("Source NAT is disabled");
             return;
             return;
         }
         }
-        openstackService.ports(routerInterfaceMap.get(routerInterface.portId()).value()).forEach(p -> {
-                    Ip4Address vmIp = (Ip4Address) p.fixedIps().values().toArray()[0];
-                    if (newList == null) {
-                        rulePopulator.removeL3Rules(vmIp,
-                                getL3ConnectionList(p.networkId(), getOpenstackRouterInterface(router)));
-                    } else {
-                        rulePopulator.removeL3Rules(vmIp, newList);
-                    }
-                }
-        );
-    }
 
 
-    @Override
-    public String networkIdForRouterInterface(String portId) {
-        return routerInterfaceMap.get(portId).value();
+        // FIXME router interface is subnet specific, not network
+        OpenstackNetwork osNet = openstackService.network(osNetId);
+        populateExternalRules(osNet);
     }
     }
 
 
-    private Collection<OpenstackFloatingIP> associatedFloatingIps() {
-        List<OpenstackFloatingIP> fIps = Lists.newArrayList();
-        floatingIpMap.values()
-                .stream()
-                .filter(fIp -> fIp.value().portId() != null)
-                .forEach(fIp -> fIps.add(fIp.value()));
-        return fIps;
-    }
-
-    private void reloadInitL3Rules() {
-
-        l3EventExecutorService.execute(() ->
-                openstackService.ports()
-                        .stream()
-                        .forEach(p ->
-                        {
-                            if (p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)) {
-                                updateRouterInterface(portToRouterInterface(p));
-                            } else {
-                                Optional<Ip4Address> vmIp = p.fixedIps().values().stream().findAny();
-                                if (vmIp.isPresent()) {
-                                    OpenstackFloatingIP floatingIP = getOpenstackFloatingIp(vmIp.get());
-                                    if (floatingIP != null) {
-                                        updateFloatingIP(floatingIP);
-                                    }
-                                }
-                            }
-                        })
-        );
-    }
-
-    private OpenstackRouterInterface portToRouterInterface(OpenstackPort p) {
-        OpenstackRouterInterface.Builder osBuilder = new OpenstackRouterInterface.Builder()
-                .id(checkNotNull(p.deviceId()))
-                .tenantId(checkNotNull(openstackService.network(p.networkId()).tenantId()))
-                .subnetId(checkNotNull(p.fixedIps().keySet().stream().findFirst().orElse(null)).toString())
-                .portId(checkNotNull(p.id()));
-
-        return osBuilder.build();
-    }
-
-    private class InternalPacketProcessor implements PacketProcessor {
-
-        @Override
-        public void process(PacketContext context) {
-
-            DeviceId senderDeviceId = context.inPacket().receivedFrom().deviceId();
-            if (!nodeService.routerBridge(senderDeviceId).isPresent()) {
-                log.warn("No router bridge for {} is found.", senderDeviceId);
-                return;
-            }
-            if (context.isHandled()) {
-                return;
-            } else if (!checkGatewayNode(context.inPacket().receivedFrom().deviceId())) {
-                return;
-            }
-
-            InboundPacket pkt = context.inPacket();
-            Ethernet ethernet = pkt.parsed();
-
-            //TODO: Considers IPv6 later.
-            if (ethernet == null) {
-                return;
-            } else if (ethernet.getEtherType() == Ethernet.TYPE_IPV4) {
-                IPv4 iPacket = (IPv4) ethernet.getPayload();
-                switch (iPacket.getProtocol()) {
-                    case IPv4.PROTOCOL_ICMP:
-
-                        icmpEventExecutorService.execute(() ->
-                                openstackIcmpHandler.processIcmpPacket(context, ethernet));
-                        break;
-                    case IPv4.PROTOCOL_UDP:
-                        // don't process DHCP
-                        UDP udpPacket = (UDP) iPacket.getPayload();
-                        if (udpPacket.getDestinationPort() == UDP.DHCP_SERVER_PORT &&
-                                udpPacket.getSourcePort() == UDP.DHCP_CLIENT_PORT) {
-                            break;
-                        }
-                    default:
-                        int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
-                        DeviceId deviceId = pkt.receivedFrom().deviceId();
-                        Port port = null;
-                        port = deviceService.getPort(deviceId,
-                                gatewayService.getUplinkPort(deviceId));
-                        if (port != null) {
-                            OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
-                                    Ip4Address.valueOf(iPacket.getSourceAddress()));
-                            l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
-                                    portNum, openstackPort, port));
-
-                        } else {
-                            log.warn("There`s no external interface");
-                        }
-
-                        break;
-                }
-            } else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
-                arpEventExecutorService.execute(() ->
-                        openstackArpHandler.processArpPacketFromRouter(context, ethernet));
-            }
-        }
-
-        private int getPortNum(MacAddress sourceMac, int destinationAddress) {
-            int portNum = findUnusedPortNum();
-            if (portNum == 0) {
-                clearPortNumMap();
-                portNum = findUnusedPortNum();
-            }
-            tpPortNumMap.put(portNum, sourceMac.toString().concat(COLON).concat(String.valueOf(destinationAddress)));
-            return portNum;
-        }
-
-        private int findUnusedPortNum() {
-            for (int i = TP_PORT_MINIMUM_NUM; i < TP_PORT_MAXIMUM_NUM; i++) {
-                if (!tpPortNumMap.containsKey(i)) {
-                    return i;
-                }
-            }
-            return 0;
+    private void unsetExternalConnection(OpenstackRouter osRouter, String osNetId) {
+        if (!osRouter.gatewayExternalInfo().isEnablePnat()) {
+            log.debug("Source NAT is disabled");
+            return;
         }
         }
 
 
+        // FIXME router interface is subnet specific, not network
+        OpenstackNetwork osNet = openstackService.network(osNetId);
+        removeExternalRules(osNet);
     }
     }
 
 
-    private boolean checkGatewayNode(DeviceId deviceId) {
-        return gatewayService.getGatewayDeviceIds().contains(deviceId);
-    }
-
-    private void clearPortNumMap() {
-        tpPortNumMap.entrySet().forEach(e -> {
-            if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
-                tpPortNumMap.remove(e.getKey());
-            }
-        });
-    }
-
-    private Optional<Port> getExternalPort(DeviceId deviceId, String interfaceName) {
-        return deviceService.getPorts(deviceId)
-                .stream()
-                .filter(p -> p.annotations().value(PORT_NAME).equals(interfaceName))
-                .findAny();
-    }
-
-    private void checkExternalConnection(OpenstackRouter router,
-                                         Collection<OpenstackRouterInterface> interfaces) {
-        checkNotNull(router, "Router can not be null");
-        checkNotNull(interfaces, "routerInterfaces can not be null");
-        Ip4Address externalIp = router.gatewayExternalInfo().externalFixedIps()
-                .values().stream().findFirst().orElse(null);
-        if ((externalIp == null) || (!router.gatewayExternalInfo().isEnablePnat())) {
-            log.debug("Not satisfied to set pnat configuration");
+    private void setRoutes(OpenstackRouter osRouter, Optional<Host> host) {
+        Set<OpenstackNetwork> routableNets = routableNetworks(osRouter.id());
+        if (routableNets.size() < 2) {
+            // no other subnet interface is connected to this router, do nothing
             return;
             return;
         }
         }
-        interfaces.forEach(this::initiateL3Rule);
-    }
 
 
-    private Optional<OpenstackRouter> getRouterfromExternalIp(Ip4Address externalIp) {
-        return getExternalRouter(true)
-                .stream()
-                .filter(r -> r.gatewayExternalInfo()
-                        .externalFixedIps()
-                        .values()
-                        .stream()
-                        .findAny()
-                        .get()
-                        .equals(externalIp))
-                .findAny();
+        // FIXME router interface is subnet specific, not network
+        Set<String> routableNetIds = routableNets.stream()
+                .map(OpenstackNetwork::id)
+                .collect(Collectors.toSet());
+
+        Set<Host> hosts = host.isPresent() ? ImmutableSet.of(host.get()) :
+                Tools.stream(hostService.getHosts())
+                        .filter(h -> routableNetIds.contains(h.annotations().value(NETWORK_ID)))
+                        .collect(Collectors.toSet());
+
+        hosts.stream().forEach(h -> populateRoutingRules(h, routableNets));
+    }
+
+    private void unsetRoutes(OpenstackRouter osRouter, OpenstackNetwork osNet) {
+        // FIXME router interface is subnet specific, not network
+        Set<OpenstackNetwork> routableNets = routableNetworks(osRouter.id());
+        Tools.stream(hostService.getHosts())
+                .filter(h -> Objects.equals(
+                        h.annotations().value(NETWORK_ID), osNet.id()))
+                .forEach(h -> removeRoutingRules(h, routableNets));
+
+        routableNets.stream().forEach(n -> {
+            Tools.stream(hostService.getHosts())
+                    .filter(h -> Objects.equals(
+                            h.annotations().value(NETWORK_ID),
+                            n.id()))
+                    .forEach(h -> removeRoutingRules(h, ImmutableSet.of(osNet)));
+            log.debug("Removed between {} to {}", n.name(), osNet.name());
+        });
     }
     }
 
 
-    private void initiateL3Rule(OpenstackRouterInterface routerInterface) {
-        long vni = Long.parseLong(openstackService.network(openstackService
-                .port(routerInterface.portId()).networkId()).segmentId());
-        rulePopulator.populateExternalRules(vni);
+    private OpenstackRouter openstackRouter(String routerId) {
+        return openstackService.routers().stream().filter(r ->
+                r.id().equals(routerId)).iterator().next();
     }
     }
 
 
-    private Collection<OpenstackRouterInterface> getOpenstackRouterInterface(OpenstackRouter router) {
-        List<OpenstackRouterInterface> interfaces = Lists.newArrayList();
-        openstackService.ports()
-                .stream()
-                .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
-                        && p.deviceId().equals(router.id()))
-                .forEach(p -> interfaces.add(portToRouterInterface(p)));
-        return interfaces;
+    private Optional<OpenstackPort> routerIfacePort(String osNetId) {
+        // FIXME router interface is subnet specific, not network
+        return openstackService.ports().stream()
+                .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+                        p.networkId().equals(osNetId))
+                .findAny();
     }
     }
 
 
-    private OpenstackRouter getOpenstackRouter(String id) {
-        return openstackService.routers().stream().filter(r ->
-                r.id().equals(id)).iterator().next();
+    private Set<OpenstackNetwork> routableNetworks(String osRouterId) {
+        // FIXME router interface is subnet specific, not network
+        return openstackService.ports().stream()
+                .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+                        p.deviceId().equals(osRouterId))
+                .map(p -> openstackService.network(p.networkId()))
+                .collect(Collectors.toSet());
     }
     }
 
 
-    private OpenstackPort getOpenstackPort(MacAddress sourceMac, Ip4Address ip4Address) {
-        OpenstackPort openstackPort = openstackService.ports().stream()
-                .filter(p -> p.macAddress().equals(sourceMac)).iterator().next();
-        return openstackPort.fixedIps().values().stream().filter(ip ->
-                ip.equals(ip4Address)).count() > 0 ? openstackPort : null;
+    private void populateExternalRules(OpenstackNetwork osNet) {
+        populateCnodeToGateway(Long.valueOf(osNet.segmentId()));
+        populateGatewayToController(Long.valueOf(osNet.segmentId()));
     }
     }
 
 
-    private OpenstackFloatingIP getOpenstackFloatingIp(Ip4Address vmIp) {
-        Optional<OpenstackFloatingIP> floatingIp = floatingIpMap.asJavaMap().values().stream()
-                .filter(f -> f.portId() != null && f.fixedIpAddress().equals(vmIp))
-                .findAny();
+    private void removeExternalRules(OpenstackNetwork osNet) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(Long.valueOf(osNet.segmentId()))
+                .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
 
 
-        if (floatingIp.isPresent()) {
-            return floatingIp.get();
-        }
-        log.debug("There is no floating IP information for VM IP {}", vmIp);
+        nodeService.completeNodes().stream().forEach(node -> {
+            ForwardingObjective.Flag flag = node.type().equals(GATEWAY) ?
+                    ForwardingObjective.Flag.VERSATILE :
+                    ForwardingObjective.Flag.SPECIFIC;
 
 
-        return null;
-    }
-
-    private Optional<OpenstackPort> getRouterInterfacePort(String networkId) {
-
-        return openstackService.ports()
-                .stream()
-                .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
-                        && p.networkId().equals(networkId))
-                .findAny();
+            RulePopulatorUtil.removeRule(
+                    flowObjectiveService,
+                    appId,
+                    node.intBridge(),
+                    sBuilder.build(),
+                    flag,
+                    ROUTING_RULE_PRIORITY);
+        });
     }
     }
 
 
-    // TODO: Remove the function and the related codes when vRouter is running on different ONOS instance.
-    private void registerFloatingIpToHostService(OpenstackFloatingIP openstackFloatingIp, Action action) {
-
-        Optional<Host> hostOptional = hostService.getHostsByIp(openstackFloatingIp.fixedIpAddress())
-                .stream()
-                .findFirst();
-        if (!hostOptional.isPresent()) {
-            log.warn("No host with IP {} is registered and cannot add the floating IP. ",
-                    openstackFloatingIp.floatingIpAddress());
+    private void populateRoutingRules(Host host, Set<OpenstackNetwork> osNets) {
+        String osNetId = host.annotations().value(NETWORK_ID);
+        if (osNetId == null) {
             return;
             return;
         }
         }
 
 
-        Host host = hostOptional.get();
-        Set<IpAddress> ipAddresses = Sets.newHashSet();
-        if (action == Action.ASSOCIATE) {
-            ipAddresses.add(openstackFloatingIp.floatingIpAddress());
+        DeviceId localDevice = host.location().deviceId();
+        PortNumber localPort = host.location().port();
+        if (!nodeService.dataIp(localDevice).isPresent()) {
+            log.warn("Failed to populate L3 rules");
+            return;
         }
         }
 
 
-        HostDescription hostDescription =
-                new DefaultHostDescription(host.mac(), host.vlan(), host.location(), ipAddresses,
-                        (DefaultAnnotations) host.annotations());
-
-        hostProviderService.hostDetected(host.id(), hostDescription, false);
+        // TODO improve pipeline, do we have to install access rules between networks
+        // for every single VMs?
+        osNets.stream().filter(osNet -> !osNet.id().equals(osNetId)).forEach(osNet -> {
+            populateRoutingRulestoSameNode(
+                    host.ipAddresses().stream().findFirst().get().getIp4Address(),
+                    host.mac(),
+                    localPort, localDevice,
+                    Long.valueOf(osNet.segmentId()));
+
+            nodeService.completeNodes().stream()
+                    .filter(node -> node.type().equals(COMPUTE))
+                    .filter(node -> !node.intBridge().equals(localDevice))
+                    .forEach(node -> populateRoutingRulestoDifferentNode(
+                            host.ipAddresses().stream().findFirst().get().getIp4Address(),
+                            Long.valueOf(osNet.segmentId()),
+                            node.intBridge(),
+                            nodeService.dataIp(localDevice).get().getIp4Address()));
+        });
     }
     }
 
 
-    private class InternalHostListener implements HostListener {
-
-        private void hostDetected(Host host) {
-            String portId = host.annotations().value(Constants.PORT_ID);
-            OpenstackPort openstackPort = openstackService.port(portId);
-            if (openstackPort == null) {
-                log.warn("No OpenstackPort information found from OpenStack for port ID {}", portId);
-                return;
-            }
-
-            Optional<OpenstackPort> routerPort = getRouterInterfacePort(openstackPort.networkId());
-            if (routerPort.isPresent()) {
-                OpenstackRouterInterface routerInterface = portToRouterInterface(routerPort.get());
-                l3EventExecutorService.execute(() ->
-                        setL3Connection(getOpenstackRouter(routerInterface.id()), openstackPort));
-
-            }
+    private void removeRoutingRules(Host host, Set<OpenstackNetwork> osNets) {
+        String osNetId = host.annotations().value(NETWORK_ID);
+        if (osNetId == null) {
+            return;
         }
         }
 
 
-        private void hostRemoved(Host host) {
-            String portId = host.annotations().value(Constants.PORT_ID);
-            OpenstackPort openstackPort = openstackService.port(portId);
-            if (openstackPort == null) {
-                log.warn("No OpenstackPort information found from OpenStack for port ID {}", portId);
-                return;
-            }
+        osNets.stream().filter(osNet -> !osNet.id().equals(osNetId)).forEach(osNet -> {
+            TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+            sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                    .matchIPDst(host.ipAddresses().stream().findFirst().get().toIpPrefix())
+                    .matchTunnelId(Long.valueOf(osNet.segmentId()));
+
+            nodeService.completeNodes().stream()
+                    .filter(node -> node.type().equals(COMPUTE))
+                    .forEach(node -> RulePopulatorUtil.removeRule(
+                            flowObjectiveService,
+                            appId,
+                            node.intBridge(),
+                            sBuilder.build(),
+                            ForwardingObjective.Flag.SPECIFIC,
+                            ROUTING_RULE_PRIORITY));
+        });
+        log.debug("Removed routing rule from {} to {}", host, osNets);
+    }
 
 
-            Optional<OpenstackPort> routerPort = getRouterInterfacePort(openstackPort.networkId());
-            if (routerPort.isPresent()) {
-                OpenstackRouterInterface routerInterface = portToRouterInterface(routerPort.get());
-                IpAddress ipAddress = host.ipAddresses().stream().findFirst().get();
-                l3EventExecutorService.execute(() -> rulePopulator.removeL3Rules(ipAddress.getIp4Address(),
-                        getL3ConnectionList(host.annotations().value(Constants.NETWORK_ID),
-                                getOpenstackRouterInterface(getOpenstackRouter(routerInterface.id())))));
-            }
-        }
+    private void populateGatewayToController(long vni) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(vni)
+                .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+        tBuilder.setOutput(PortNumber.CONTROLLER);
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withFlag(ForwardingObjective.Flag.VERSATILE)
+                .withPriority(ROUTING_RULE_PRIORITY)
+                .fromApp(appId)
+                .add();
+
+        gatewayService.getGatewayDeviceIds().stream()
+                .forEach(deviceId -> flowObjectiveService.forward(deviceId, fo));
+    }
+
+    private void populateCnodeToGateway(long vni) {
+        nodeService.completeNodes().stream()
+                .filter(node -> node.type().equals(COMPUTE))
+                .forEach(node -> populateRuleToGateway(
+                        node.intBridge(),
+                        gatewayService.getGatewayGroupId(node.intBridge()),
+                        vni));
+    }
+
+    private void populateRuleToGateway(DeviceId deviceId, GroupId groupId, long vni) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(vni)
+                .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+
+        tBuilder.group(groupId);
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .withPriority(ROUTING_RULE_PRIORITY)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(deviceId, fo);
+    }
+
+    private void populateRoutingRulestoDifferentNode(Ip4Address vmIp, long vni,
+                                                     DeviceId deviceId, Ip4Address hostIp) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(vni)
+                .matchIPDst(vmIp.toIpPrefix());
+        tBuilder.extension(buildExtension(deviceService, deviceId, hostIp), deviceId)
+                .setOutput(nodeService.tunnelPort(deviceId).get());
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(ROUTING_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(deviceId, fo);
+    }
+
+    private void populateRoutingRulestoSameNode(Ip4Address vmIp, MacAddress vmMac,
+                                                PortNumber port, DeviceId deviceId, long vni) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(vmIp.toIpPrefix())
+                .matchTunnelId(vni);
+
+        tBuilder.setEthDst(vmMac)
+                .setOutput(port);
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(ROUTING_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(deviceId, fo);
+    }
+
+    private void reloadRoutingRules() {
+        eventExecutor.execute(() -> openstackService.ports().stream()
+                .filter(osPort -> osPort.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
+                .forEach(osPort -> {
+                    OpenstackRouter osRouter = openstackRouter(osPort.deviceId());
+                    setRoutes(osRouter, Optional.empty());
+                    if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+                        setExternalConnection(osRouter, osPort.networkId());
+                    }
+                }));
+    }
 
 
-        private boolean isValidHost(Host host) {
-            return !host.ipAddresses().isEmpty() &&
-                    host.annotations().value(Constants.VXLAN_ID) != null &&
-                    host.annotations().value(Constants.NETWORK_ID) != null &&
-                    host.annotations().value(Constants.TENANT_ID) != null &&
-                    host.annotations().value(Constants.PORT_ID) != null;
+    @Override
+    protected void hostDetected(Host host) {
+        String osNetId = host.annotations().value(NETWORK_ID);
+        Optional<OpenstackPort> routerIface = routerIfacePort(osNetId);
+        if (!routerIface.isPresent()) {
+            return;
         }
         }
+        eventExecutor.execute(() -> setRoutes(
+                openstackRouter(routerIface.get().deviceId()),
+                Optional.of(host)));
+    }
 
 
-        @Override
-        public void event(HostEvent event) {
-            Host host = event.subject();
-            if (!mastershipService.isLocalMaster(host.location().deviceId())) {
-                // do not allow to proceed without mastership
-                return;
-            }
-
-            if (!isValidHost(host)) {
-                log.debug("Invalid host event, ignore it {}", host);
-                return;
-            }
-
-            switch (event.type()) {
-                case HOST_UPDATED:
-                case HOST_ADDED:
-                    l3EventExecutorService.execute(() -> hostDetected(host));
-                    break;
-                case HOST_REMOVED:
-                    l3EventExecutorService.execute(() -> hostRemoved(host));
-                    break;
-                default:
-                    break;
-            }
+    @Override
+    protected void hostRemoved(Host host) {
+        String osNetId = host.annotations().value(NETWORK_ID);
+        Optional<OpenstackPort> routerIface = routerIfacePort(osNetId);
+        if (!routerIface.isPresent()) {
+            return;
         }
         }
+        Set<OpenstackNetwork> routableNets = routableNetworks(routerIface.get().deviceId());
+        eventExecutor.execute(() -> removeRoutingRules(host, routableNets));
     }
     }
 
 
-    private class InternalOpenstackNodeListener implements OpenstackNodeListener {
-
-        private void nodeComplete() {
-
-            rulePopulator = new OpenstackRoutingRulePopulator(appId, openstackService, flowObjectiveService,
-                    deviceService, driverService, nodeService, gatewayService);
-            openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService, hostService,
-                    openstackService, nodeService, gatewayService);
-            openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService, nodeService,
-                    gatewayService);
-
-            // Packet handlers must be started AFTER all initialization processes.
-            packetService.addProcessor(internalPacketProcessor, PacketProcessor.director(1));
-
-            openstackIcmpHandler.requestPacket(appId);
-            openstackArpHandler.requestPacket(appId);
-
-            openstackService.floatingIps().stream()
-                    .forEach(f -> floatingIpMap.put(f.id(), f));
-
-            reloadInitL3Rules();
-        }
+    private class InternalNodeListener implements OpenstackNodeListener {
 
 
         @Override
         @Override
         public void event(OpenstackNodeEvent event) {
         public void event(OpenstackNodeEvent event) {
@@ -730,28 +481,22 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
             switch (event.type()) {
             switch (event.type()) {
                 case COMPLETE:
                 case COMPLETE:
                     log.info("COMPLETE node {} detected", node.hostname());
                     log.info("COMPLETE node {} detected", node.hostname());
-                    l3EventExecutorService.execute(() -> nodeComplete());
+                    if (node.type() == GATEWAY) {
+                        GatewayNode gnode = GatewayNode.builder()
+                                .gatewayDeviceId(node.intBridge())
+                                .dataIpAddress(node.dataIp().getIp4Address())
+                                .uplinkIntf(node.externalPortName().get())
+                                .build();
+                        gatewayService.addGatewayNode(gnode);
+                    }
+                    eventExecutor.execute(OpenstackRoutingManager.this::reloadRoutingRules);
                     break;
                     break;
+                case INIT:
+                case DEVICE_CREATED:
                 case INCOMPLETE:
                 case INCOMPLETE:
-                    break;
                 default:
                 default:
                     break;
                     break;
             }
             }
         }
         }
     }
     }
-
-    private class InternalHostProvider extends AbstractProvider implements HostProvider {
-
-        /**
-         * Creates a provider with the supplier identifier.
-         */
-        protected InternalHostProvider() {
-            super(PID);
-        }
-
-        @Override
-        public void triggerProbe(Host host) {
-            // nothing to do
-        }
-    }
 }
 }

-/*
- * Copyright 2016-present Open Networking Laboratory
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.onosproject.openstacknetworking.routing;
-
-import com.google.common.collect.Lists;
-import org.onlab.packet.Ethernet;
-import org.onlab.packet.IPv4;
-import org.onlab.packet.Ip4Address;
-import org.onlab.packet.IpAddress;
-import org.onlab.packet.IpPrefix;
-import org.onlab.packet.MacAddress;
-import org.onlab.packet.TCP;
-import org.onlab.packet.TpPort;
-import org.onlab.packet.UDP;
-import org.onosproject.core.ApplicationId;
-import org.onosproject.core.GroupId;
-import org.onosproject.net.Device;
-import org.onosproject.net.DeviceId;
-import org.onosproject.net.Host;
-import org.onosproject.net.Port;
-import org.onosproject.net.PortNumber;
-import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
-import org.onosproject.net.device.DeviceService;
-import org.onosproject.net.driver.DefaultDriverData;
-import org.onosproject.net.driver.DefaultDriverHandler;
-import org.onosproject.net.driver.Driver;
-import org.onosproject.net.driver.DriverHandler;
-import org.onosproject.net.driver.DriverService;
-import org.onosproject.net.flow.DefaultTrafficSelector;
-import org.onosproject.net.flow.DefaultTrafficTreatment;
-import org.onosproject.net.flow.TrafficSelector;
-import org.onosproject.net.flow.TrafficTreatment;
-import org.onosproject.net.flow.instructions.ExtensionPropertyException;
-import org.onosproject.net.flow.instructions.ExtensionTreatment;
-import org.onosproject.net.flow.instructions.ExtensionTreatmentType;
-import org.onosproject.net.flowobjective.DefaultForwardingObjective;
-import org.onosproject.net.flowobjective.FlowObjectiveService;
-import org.onosproject.net.flowobjective.ForwardingObjective;
-import org.onosproject.net.packet.InboundPacket;
-import org.onosproject.openstackinterface.OpenstackInterfaceService;
-import org.onosproject.openstackinterface.OpenstackPort;
-import org.onosproject.openstackinterface.OpenstackRouterInterface;
-import org.onosproject.openstackinterface.OpenstackSubnet;
-import org.onosproject.openstackinterface.OpenstackFloatingIP;
-import org.onosproject.openstacknetworking.Constants;
-import org.onosproject.openstacknetworking.OpenstackRoutingService;
-import org.onosproject.scalablegateway.api.ScalableGatewayService;
-import org.onosproject.openstacknode.OpenstackNode;
-import org.onosproject.openstacknode.OpenstackNodeService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.StreamSupport;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.onlab.osgi.DefaultServiceDirectory.getService;
-import static org.onosproject.net.AnnotationKeys.PORT_NAME;
-
-/**
- * Populates Routing Flow Rules.
- */
-public class OpenstackRoutingRulePopulator {
-
-    private final Logger log = LoggerFactory.getLogger(getClass());
-
-    private final ApplicationId appId;
-    private final FlowObjectiveService flowObjectiveService;
-    private final OpenstackInterfaceService openstackService;
-    private final DeviceService deviceService;
-    private final DriverService driverService;
-    private final ScalableGatewayService gatewayService;
-    private final OpenstackNodeService nodeService;
-
-    private static final String PORTNAME_PREFIX_TUNNEL = "vxlan";
-    private static final String PORTNAME_PREFIX_VM = "tap";
-
-    private static final String PORTNOTNULL = "Port can not be null";
-    private static final String DEVICENOTNULL = "Device can not be null";
-    private static final String TUNNEL_DESTINATION = "tunnelDst";
-    private static final int ROUTING_RULE_PRIORITY = 25000;
-    private static final int FLOATING_RULE_PRIORITY = 42000;
-    private static final int PNAT_RULE_PRIORITY = 26000;
-    private static final int PNAT_TIMEOUT = 120;
-    private static final int PREFIX_LENGTH = 32;
-
-    private InboundPacket inboundPacket;
-    private OpenstackPort openstackPort;
-    private int portNum;
-    private MacAddress externalInterface;
-    private MacAddress externalRouter;
-
-    /**
-     * The constructor of openstackRoutingRulePopulator.
-     *
-     * @param appId Caller`s appId
-     * @param openstackService Opestack REST request handler
-     * @param flowObjectiveService FlowObjectiveService
-     * @param deviceService DeviceService
-     * @param driverService DriverService
-     * @param nodeService openstack node service
-     * @param gatewayService scalable gateway service
-     */
-    public OpenstackRoutingRulePopulator(ApplicationId appId,
-                                         OpenstackInterfaceService openstackService,
-                                         FlowObjectiveService flowObjectiveService,
-                                         DeviceService deviceService,
-                                         DriverService driverService,
-                                         OpenstackNodeService nodeService,
-                                         ScalableGatewayService gatewayService) {
-        this.appId = appId;
-        this.flowObjectiveService = flowObjectiveService;
-        this.openstackService = checkNotNull(openstackService);
-        this.deviceService = deviceService;
-        this.driverService = driverService;
-        this.gatewayService = gatewayService;
-        this.nodeService = nodeService;
-    }
-
-    /**
-     * Populates flow rules for Pnat configurations.
-     *
-     * @param inboundPacket Packet-in event packet
-     * @param openstackPort Target VM information
-     * @param portNum Pnat port number
-     * @param externalIp external ip address
-     * @param externalInterfaceMacAddress Gateway external interface macaddress
-     * @param externalRouterMacAddress Outer(physical) router`s macaddress
-     */
-    public void populatePnatFlowRules(InboundPacket inboundPacket, OpenstackPort openstackPort, int portNum,
-                                      Ip4Address externalIp, MacAddress externalInterfaceMacAddress,
-                                      MacAddress externalRouterMacAddress) {
-        this.inboundPacket = inboundPacket;
-        this.openstackPort = openstackPort;
-        this.portNum = portNum;
-        this.externalInterface = externalInterfaceMacAddress;
-        this.externalRouter = externalRouterMacAddress;
-
-        long vni = getVni(openstackPort.networkId());
-
-        populatePnatIncomingFlowRules(vni, externalIp);
-        populatePnatOutgoingFlowRules(vni, externalIp);
-    }
-
-    private void populatePnatOutgoingFlowRules(long vni, Ip4Address externalIp) {
-        IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
-
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchIPProtocol(iPacket.getProtocol())
-                .matchTunnelId(vni)
-                .matchIPSrc(IpPrefix.valueOf(iPacket.getSourceAddress(), 32))
-                .matchIPDst(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
-
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        switch (iPacket.getProtocol()) {
-            case IPv4.PROTOCOL_TCP:
-                TCP tcpPacket = (TCP) iPacket.getPayload();
-                sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getSourcePort()))
-                        .matchTcpDst(TpPort.tpPort(tcpPacket.getDestinationPort()));
-                tBuilder.setTcpSrc(TpPort.tpPort(portNum));
-                break;
-            case IPv4.PROTOCOL_UDP:
-                UDP udpPacket = (UDP) iPacket.getPayload();
-                sBuilder.matchUdpSrc(TpPort.tpPort(udpPacket.getSourcePort()))
-                        .matchUdpDst(TpPort.tpPort(udpPacket.getDestinationPort()));
-                tBuilder.setUdpSrc(TpPort.tpPort(portNum));
-                break;
-            default:
-                log.debug("Unsupported IPv4 protocol {}");
-                break;
-        }
-
-        tBuilder.setIpSrc(externalIp);
-        gatewayService.getGatewayNodes().forEach(node -> {
-            tBuilder.setOutput(gatewayService.getUplinkPort(node.getGatewayDeviceId()));
-            ForwardingObjective fo = DefaultForwardingObjective.builder()
-                    .withSelector(sBuilder.build())
-                    .withTreatment(tBuilder.build())
-                    .withFlag(ForwardingObjective.Flag.VERSATILE)
-                    .withPriority(PNAT_RULE_PRIORITY)
-                    .makeTemporary(PNAT_TIMEOUT)
-                    .fromApp(appId)
-                    .add();
-
-            flowObjectiveService.forward(node.getGatewayDeviceId(), fo);
-        });
-
-    }
-
-    private Port getPortOfExternalInterface() {
-        return deviceService.getPorts(getGatewayNode().id()).stream()
-                .filter(p -> p.annotations().value(PORT_NAME)
-                        .equals(org.onosproject.openstacknode.Constants.PATCH_INTG_BRIDGE))
-                .findAny().orElse(null);
-    }
-
-
-    private void populatePnatIncomingFlowRules(long vni, Ip4Address externalIp) {
-        IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
-
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchIPProtocol(iPacket.getProtocol())
-                .matchIPDst(IpPrefix.valueOf(externalIp, 32))
-                .matchIPSrc(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
-
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-        tBuilder.setTunnelId(vni)
-                .setEthDst(inboundPacket.parsed().getSourceMAC())
-                .setIpDst(IpAddress.valueOf(iPacket.getSourceAddress()));
-
-        switch (iPacket.getProtocol()) {
-            case IPv4.PROTOCOL_TCP:
-                TCP tcpPacket = (TCP) iPacket.getPayload();
-                sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getDestinationPort()))
-                        .matchTcpDst(TpPort.tpPort(portNum));
-                tBuilder.setTcpDst(TpPort.tpPort(tcpPacket.getSourcePort()));
-                break;
-            case IPv4.PROTOCOL_UDP:
-                UDP udpPacket = (UDP) iPacket.getPayload();
-                sBuilder.matchUdpSrc(TpPort.tpPort(udpPacket.getDestinationPort()))
-                        .matchUdpDst(TpPort.tpPort(portNum));
-                tBuilder.setUdpDst(TpPort.tpPort(udpPacket.getSourcePort()));
-                break;
-            default:
-                break;
-        }
-
-        getGatewayNodeList().forEach(node -> {
-            DeviceId deviceId = node.id();
-            tBuilder.extension(buildNiciraExtenstion(deviceId,
-                    getHostIpfromOpenstackPort(openstackPort).getIp4Address()), deviceId)
-                    .setOutput(getTunnelPort(deviceId));
-
-            ForwardingObjective fo = DefaultForwardingObjective.builder()
-                    .withSelector(sBuilder.build())
-                    .withTreatment(tBuilder.build())
-                    .withFlag(ForwardingObjective.Flag.VERSATILE)
-                    .withPriority(PNAT_RULE_PRIORITY)
-                    .makeTemporary(PNAT_TIMEOUT)
-                    .fromApp(appId)
-                    .add();
-
-            flowObjectiveService.forward(deviceId, fo);
-        });
-    }
-
-    private List<Device> getGatewayNodeList() {
-        List<Device> devices = Lists.newArrayList();
-        gatewayService.getGatewayDeviceIds().forEach(deviceId ->
-                devices.add(checkNotNull(deviceService.getDevice(deviceId))));
-        return devices;
-    }
-
-    private IpAddress getHostIpfromOpenstackPort(OpenstackPort openstackPort) {
-        Device device = getDevicefromOpenstackPort(openstackPort);
-
-        Optional<IpAddress> ipAddress = nodeService.dataIp(device.id());
-        if (!ipAddress.isPresent()) {
-            log.warn("No IP address found for device {}", device.id());
-            return null;
-        }
-
-        return ipAddress.get();
-    }
-
-    private Device getDevicefromOpenstackPort(OpenstackPort openstackPort) {
-        String openstackPortName = PORTNAME_PREFIX_VM + openstackPort.id().substring(0, 11);
-        Device device = StreamSupport.stream(deviceService.getDevices().spliterator(), false)
-                .filter(d -> findPortinDevice(d.id(), openstackPortName))
-                .iterator()
-                .next();
-        checkNotNull(device, DEVICENOTNULL);
-        return device;
-    }
-
-    private boolean findPortinDevice(DeviceId deviceId, String openstackPortName) {
-        Port port = deviceService.getPorts(deviceId)
-                .stream()
-                .filter(p -> p.isEnabled() && p.annotations().value(PORT_NAME).equals(openstackPortName))
-                .findAny()
-                .orElse(null);
-        return port != null;
-    }
-
-    /**
-     * Builds Nicira extension for tagging remoteIp of vxlan.
-     *
-     * @param deviceId Device Id of vxlan source device
-     * @param hostIp Remote Ip of vxlan destination device
-     * @return NiciraExtension Treatment
-     */
-    public ExtensionTreatment buildNiciraExtenstion(DeviceId deviceId, Ip4Address hostIp) {
-        Driver driver = driverService.getDriver(deviceId);
-        DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, deviceId));
-        ExtensionTreatmentResolver resolver = driverHandler.behaviour(ExtensionTreatmentResolver.class);
-
-        ExtensionTreatment extensionInstruction =
-                resolver.getExtensionInstruction(
-                        ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST.type());
-
-        try {
-            extensionInstruction.setPropertyValue(TUNNEL_DESTINATION, hostIp);
-        } catch (ExtensionPropertyException e) {
-            log.error("Error setting Nicira extension setting {}", e);
-        }
-
-        return extensionInstruction;
-    }
-
-    /**
-     * Returns port number of vxlan tunnel.
-     *
-     * @param deviceId Target Device Id
-     * @return PortNumber
-     */
-    public PortNumber getTunnelPort(DeviceId deviceId) {
-        Port port = deviceService.getPorts(deviceId).stream()
-                .filter(p -> p.annotations().value(PORT_NAME).equals(PORTNAME_PREFIX_TUNNEL))
-                .findAny().orElse(null);
-
-        if (port == null) {
-            log.error("No TunnelPort was created.");
-            return null;
-        }
-        return port.number();
-
-    }
-
-    /**
-     * Populates flow rules from openstackComputeNode to GatewayNode.
-     *
-     * @param vni Target network
-     */
-    public void populateExternalRules(long vni) {
-
-        // 1. computeNode to gateway
-        populateComputeNodeRules(vni);
-        // 2. gatewayNode to controller
-        populateRuleGatewaytoController(vni);
-    }
-
-    private void populateRuleGatewaytoController(long vni) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchTunnelId(vni)
-                .matchEthDst(Constants.GATEWAY_MAC);
-        tBuilder.setOutput(PortNumber.CONTROLLER);
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withFlag(ForwardingObjective.Flag.VERSATILE)
-                .withPriority(ROUTING_RULE_PRIORITY)
-                .fromApp(appId)
-                .add();
-
-        getGatewayNodeList().forEach(device -> flowObjectiveService.forward(device.id(), fo));
-    }
-
-    private void populateComputeNodeRules(long vni) {
-        StreamSupport.stream(deviceService.getDevices().spliterator(), false)
-                .filter(d -> isTypeOf(d.id(), OpenstackNodeService.NodeType.COMPUTE))
-                .forEach(d -> populateRuleToGatewayBySgw(d.id(),
-                        gatewayService.getGatewayGroupId(d.id()), vni));
-    }
-
-    private void populateRuleToGatewayBySgw(DeviceId deviceId, GroupId groupId, long vni) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchTunnelId(vni)
-                .matchEthDst(Constants.GATEWAY_MAC);
-
-        tBuilder.group(groupId);
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .withPriority(ROUTING_RULE_PRIORITY)
-                .fromApp(appId)
-                .add();
-
-        flowObjectiveService.forward(deviceId, fo);
-    }
-
-    private Device getGatewayNode() {
-
-        // TODO Return the correct gateway node
-        Optional<OpenstackNode> gwNode =  nodeService.nodes().stream()
-                .filter(n -> n.type().equals(OpenstackNodeService.NodeType.GATEWAY))
-                .findFirst();
-
-        if (!gwNode.isPresent()) {
-            log.warn("No Gateway is defined.");
-            return null;
-        }
-
-        return deviceService.getDevice(gwNode.get().intBridge());
-    }
-
-    private boolean isTypeOf(DeviceId deviceId, OpenstackNodeService.NodeType type) {
-
-        Optional<OpenstackNode> node = nodeService.nodes().stream()
-                .filter(n -> n.intBridge().equals(deviceId) ||
-                        (n.routerBridge().isPresent() && n.routerBridge().get().equals(deviceId)))
-                .filter(n -> n.type().equals(type))
-                .findFirst();
-
-        if (node.isPresent()) {
-            return true;
-        }
-
-        return false;
-    }
-
-    private long getVni(String netId) {
-        return Long.parseLong(openstackService.network(netId).segmentId());
-    }
-
-    /**
-     * Remove flow rules for external connection.
-     *
-     * @param routerInterface Corresponding routerInterface
-     */
-    public void removeExternalRules(OpenstackRouterInterface routerInterface) {
-        OpenstackSubnet openstackSubnet = openstackService.subnet(routerInterface.subnetId());
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchTunnelId(getVni(openstackSubnet.networkId()))
-                .matchEthDst(Constants.GATEWAY_MAC);
-
-        StreamSupport.stream(deviceService.getDevices().spliterator(), false)
-                .forEach(d -> {
-                    ForwardingObjective.Flag flag = isTypeOf(d.id(), OpenstackNodeService.NodeType.GATEWAY) ?
-                            ForwardingObjective.Flag.VERSATILE :
-                            ForwardingObjective.Flag.SPECIFIC;
-                    removeRule(d.id(), sBuilder, flag, ROUTING_RULE_PRIORITY);
-                });
-
-    }
-
-    private void removeRule(DeviceId deviceId, TrafficSelector.Builder sBuilder,
-                            ForwardingObjective.Flag flag, int priority) {
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withFlag(flag)
-                .withPriority(priority)
-                .fromApp(appId)
-                .remove();
-
-        flowObjectiveService.forward(deviceId, fo);
-    }
-
-    /**
-     * Populates flow rules for floatingIp configuration.
-     *
-     * @param floatingIP Corresponding floating ip information
-     */
-    public void populateFloatingIpRules(OpenstackFloatingIP floatingIP) {
-        OpenstackPort port = openstackService.port(floatingIP.portId());
-        //1. incoming rules
-        populateFloatingIpIncomingRules(floatingIP, port);
-        //2. outgoing rules
-        populateFloatingIpOutgoingRules(floatingIP, port);
-    }
-
-    private void populateFloatingIpIncomingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
-
-        DeviceId gatewayDeviceId = DeviceId.deviceId(port.deviceId());
-        Optional<IpAddress> ipAddress = nodeService.dataIp(gatewayDeviceId);
-        if (!ipAddress.isPresent()) {
-            log.warn("No IP address found for device {}", port.deviceId());
-            return;
-        }
-        tBuilder.setEthSrc(Constants.GATEWAY_MAC)
-                .setEthDst(port.macAddress())
-                .setIpDst(floatingIP.fixedIpAddress())
-                .setTunnelId(getVni(port.networkId()))
-                .extension(buildNiciraExtenstion(gatewayDeviceId,
-                        ipAddress.get().getIp4Address()), gatewayDeviceId)
-                .setOutput(getTunnelPort(gatewayDeviceId));
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withFlag(ForwardingObjective.Flag.VERSATILE)
-                .withPriority(FLOATING_RULE_PRIORITY)
-                .fromApp(appId)
-                .add();
-
-        flowObjectiveService.forward(getGatewayNode().id(), fo);
-    }
-
-    private void populateFloatingIpOutgoingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchTunnelId(getVni(port.networkId()))
-                .matchIPSrc(IpPrefix.valueOf(floatingIP.fixedIpAddress(), 32));
-
-        getGatewayNodeList().forEach(device -> {
-            DeviceId deviceId = device.id();
-            tBuilder.setIpSrc(floatingIP.floatingIpAddress())
-                    .setEthSrc(Constants.GW_EXT_INT_MAC)
-                    .setEthDst(Constants.PHY_ROUTER_MAC)
-                    .setOutput(getExternalPortNum(deviceId));
-
-            ForwardingObjective fo = DefaultForwardingObjective.builder()
-                    .withSelector(sBuilder.build())
-                    .withTreatment(tBuilder.build())
-                    .withFlag(ForwardingObjective.Flag.VERSATILE)
-                    .withPriority(FLOATING_RULE_PRIORITY)
-                    .fromApp(appId)
-                    .add();
-
-            flowObjectiveService.forward(deviceId, fo);
-        });
-    }
-
-    private PortNumber getExternalPortNum(DeviceId deviceId) {
-        return checkNotNull(gatewayService.getUplinkPort(deviceId), PORTNOTNULL);
-    }
-
-    /**
-     * Removes flow rules for floating ip configuration.
-     *
-     * @param floatingIp Corresponding floating ip information
-     * @param host host information for vm to remove
-     */
-    public void removeFloatingIpRules(OpenstackFloatingIP floatingIp, Host host) {
-        TrafficSelector.Builder sOutgoingBuilder = DefaultTrafficSelector.builder();
-        TrafficSelector.Builder sIncomingBuilder = DefaultTrafficSelector.builder();
-
-        // XXX FloatingIp.tenant_id() == host.vxlan_id ???
-        sOutgoingBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchTunnelId(Integer.parseInt(host.annotations().value(Constants.VXLAN_ID)))
-                .matchIPSrc(IpPrefix.valueOf(floatingIp.fixedIpAddress(), PREFIX_LENGTH));
-
-        sIncomingBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchIPDst(IpPrefix.valueOf(floatingIp.floatingIpAddress(), PREFIX_LENGTH));
-
-        getGatewayNodeList().forEach(device -> {
-            removeRule(device.id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
-            removeRule(device.id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
-        });
-    }
-
-    /**
-     * Populates L3 rules for east to west traffic.
-     *
-     * @param openstackPort target VM
-     * @param targetList target openstackRouterInterfaces
-     */
-    public void populateL3Rules(OpenstackPort openstackPort, List<OpenstackRouterInterface> targetList) {
-        Device device = getDevicefromOpenstackPort(openstackPort);
-        Port port = getPortFromOpenstackPort(device, openstackPort);
-        Ip4Address vmIp = openstackPort.fixedIps().values().iterator().next();
-
-        if (port == null) {
-            return;
-        }
-
-        targetList.forEach(routerInterface -> {
-            long vni = getVni(openstackService.port(routerInterface.portId()).networkId());
-
-            if (vmIp == null) {
-                return;
-            }
-
-            populateL3RulestoSameNode(vmIp, openstackPort, port, device, vni);
-
-            deviceService.getAvailableDevices().forEach(d -> {
-                if (!d.equals(device) && !d.equals(getGatewayNode())) {
-                    populateL3RulestoDifferentNode(vmIp, vni, d.id(),
-                            getHostIpfromOpenstackPort(openstackPort).getIp4Address());
-                }
-            });
-        });
-    }
-
-    private void populateL3RulestoDifferentNode(Ip4Address vmIp, long vni, DeviceId deviceId, Ip4Address hostIp) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchTunnelId(vni)
-                .matchIPDst(vmIp.toIpPrefix());
-        tBuilder.extension(buildNiciraExtenstion(deviceId, hostIp), deviceId)
-                .setOutput(getTunnelPort(deviceId));
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withPriority(ROUTING_RULE_PRIORITY)
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .fromApp(appId)
-                .add();
-
-        flowObjectiveService.forward(deviceId, fo);
-    }
-
-    private void populateL3RulestoSameNode(Ip4Address vmIp, OpenstackPort p, Port port, Device device, long vni) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchIPDst(vmIp.toIpPrefix())
-                .matchTunnelId(vni);
-
-        tBuilder.setEthDst(p.macAddress())
-                .setOutput(port.number());
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withPriority(ROUTING_RULE_PRIORITY)
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .fromApp(appId)
-                .add();
-
-        flowObjectiveService.forward(device.id(), fo);
-    }
-
-    private Port getPortFromOpenstackPort(Device device, OpenstackPort p) {
-        String openstackPortName = PORTNAME_PREFIX_VM + p.id().substring(0, 11);
-        return  deviceService.getPorts(device.id())
-                .stream()
-                .filter(pt -> pt.annotations().value(PORT_NAME).equals(openstackPortName))
-                .findAny()
-                .orElse(null);
-    }
-
-    /**
-     * Removes L3 rules for routerInterface events.
-     *
-     * @param vmIp Corresponding Vm ip
-     * @param routerInterfaces Corresponding routerInterfaces
-     */
-    public void removeL3Rules(Ip4Address vmIp, List<OpenstackRouterInterface> routerInterfaces) {
-        if (vmIp == null) {
-            return;
-        }
-
-        OpenstackRoutingService routingService = getService(OpenstackRoutingService.class);
-
-        deviceService.getAvailableDevices().forEach(d -> {
-            if (isTypeOf(d.id(), OpenstackNodeService.NodeType.COMPUTE)) {
-                routerInterfaces.forEach(routerInterface -> {
-                    String networkId = routingService.networkIdForRouterInterface(routerInterface.portId());
-                    long vni = getVni(networkId);
-
-                    TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-
-                    sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                            .matchIPDst(vmIp.toIpPrefix())
-                            .matchTunnelId(vni);
-
-                    removeRule(d.id(), sBuilder, ForwardingObjective.Flag.SPECIFIC, ROUTING_RULE_PRIORITY);
-                });
-            }
-        });
-    }
-}

@@ -15,6 +15,8 @@
  */
  */
 
 
 /**
 /**
- * Application for OpenstackRouting.
+ * Implements OpenStack L3 service plugin, which routes packets between subnets,
+ * forwards packets from internal networks to external ones, and accesses instances
+ * from external networks through floating IPs.
  */
  */
-package org.onosproject.openstacknetworking.routing;
\ No newline at end of file
+package org.onosproject.openstacknetworking.routing;

@@ -30,6 +30,7 @@ import org.onlab.packet.Ip4Address;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.TpPort;
 import org.onlab.packet.TpPort;
 import org.onlab.util.Tools;
 import org.onlab.util.Tools;
+import org.onosproject.core.ApplicationId;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.Host;
 import org.onosproject.net.Host;
 import org.onosproject.net.flow.DefaultTrafficSelector;
 import org.onosproject.net.flow.DefaultTrafficSelector;
@@ -43,6 +44,7 @@ import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstackinterface.OpenstackSecurityGroup;
 import org.onosproject.openstackinterface.OpenstackSecurityGroup;
 import org.onosproject.openstackinterface.OpenstackSecurityGroupRule;
 import org.onosproject.openstackinterface.OpenstackSecurityGroupRule;
 import org.onosproject.openstacknetworking.OpenstackSecurityGroupService;
 import org.onosproject.openstacknetworking.OpenstackSecurityGroupService;
+import org.onosproject.openstacknetworking.AbstractVmHandler;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;
 
 
@@ -60,7 +62,7 @@ import static org.onosproject.openstacknetworking.Constants.*;
  */
  */
 @Component(immediate = true)
 @Component(immediate = true)
 @Service
 @Service
-public class OpenstackSecurityGroupRulePopulator extends AbstractVmHandler
+public class OpenstackSecurityGroupManager extends AbstractVmHandler
         implements OpenstackSecurityGroupService {
         implements OpenstackSecurityGroupService {
 
 
     private final Logger log = LoggerFactory.getLogger(getClass());
     private final Logger log = LoggerFactory.getLogger(getClass());
@@ -77,10 +79,12 @@ public class OpenstackSecurityGroupRulePopulator extends AbstractVmHandler
     private static final String ETHTYPE_IPV4 = "IPV4";
     private static final String ETHTYPE_IPV4 = "IPV4";
 
 
     private final Map<Host, Set<SecurityGroupRule>> securityGroupRuleMap = Maps.newConcurrentMap();
     private final Map<Host, Set<SecurityGroupRule>> securityGroupRuleMap = Maps.newConcurrentMap();
+    private ApplicationId appId;
 
 
     @Activate
     @Activate
     protected void activate() {
     protected void activate() {
         super.activate();
         super.activate();
+        appId = coreService.registerApplication(SWITCHING_APP_ID);
     }
     }
 
 
     @Deactivate
     @Deactivate
@@ -96,7 +100,7 @@ public class OpenstackSecurityGroupRulePopulator extends AbstractVmHandler
 
 
         Optional<Host> host = getVmByPortId(osPort.id());
         Optional<Host> host = getVmByPortId(osPort.id());
         if (!host.isPresent()) {
         if (!host.isPresent()) {
-            log.warn("No host found with {}", osPort);
+            log.debug("No host found with {}", osPort.id());
             return;
             return;
         }
         }
         eventExecutor.execute(() -> updateSecurityGroupRules(host.get(), true));
         eventExecutor.execute(() -> updateSecurityGroupRules(host.get(), true));

@@ -40,6 +40,7 @@ import org.onosproject.net.packet.PacketService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
 import org.onosproject.openstackinterface.OpenstackNetwork;
 import org.onosproject.openstackinterface.OpenstackNetwork;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstackinterface.OpenstackPort;
+import org.onosproject.openstacknetworking.AbstractVmHandler;
 import org.osgi.service.component.ComponentContext;
 import org.osgi.service.component.ComponentContext;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;
@@ -54,12 +55,11 @@ import static org.onosproject.openstacknetworking.Constants.*;
  * Handles ARP packet from VMs.
  * Handles ARP packet from VMs.
  */
  */
 @Component(immediate = true)
 @Component(immediate = true)
-public final class OpenstackArpHandler extends AbstractVmHandler {
+public final class OpenstackSwitchingArpHandler extends AbstractVmHandler {
 
 
     private final Logger log = LoggerFactory.getLogger(getClass());
     private final Logger log = LoggerFactory.getLogger(getClass());
 
 
     private static final String GATEWAY_MAC = "gatewayMac";
     private static final String GATEWAY_MAC = "gatewayMac";
-    private static final String DEFAULT_GATEWAY_MAC = "1f:1f:1f:1f:1f:1f";
 
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected PacketService packetService;
     protected PacketService packetService;
@@ -67,9 +67,9 @@ public final class OpenstackArpHandler extends AbstractVmHandler {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected OpenstackInterfaceService openstackService;
     protected OpenstackInterfaceService openstackService;
 
 
-    @Property(name = GATEWAY_MAC, value = DEFAULT_GATEWAY_MAC,
+    @Property(name = GATEWAY_MAC, value = DEFAULT_GATEWAY_MAC_STR,
             label = "Fake MAC address for virtual network subnet gateway")
             label = "Fake MAC address for virtual network subnet gateway")
-    private String gatewayMac = DEFAULT_GATEWAY_MAC;
+    private String gatewayMac = DEFAULT_GATEWAY_MAC_STR;
 
 
     private final InternalPacketProcessor packetProcessor = new InternalPacketProcessor();
     private final InternalPacketProcessor packetProcessor = new InternalPacketProcessor();
     private final Set<Ip4Address> gateways = Sets.newConcurrentHashSet();
     private final Set<Ip4Address> gateways = Sets.newConcurrentHashSet();

+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.switching;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Sets;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.VlanId;
+import org.onosproject.core.CoreService;
+import org.onosproject.dhcp.DhcpService;
+import org.onosproject.dhcp.IpAssignment;
+import org.onosproject.mastership.MastershipService;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.DefaultAnnotations;
+import org.onosproject.net.Device;
+import org.onosproject.net.Host;
+import org.onosproject.net.HostId;
+import org.onosproject.net.HostLocation;
+import org.onosproject.net.Port;
+import org.onosproject.net.device.DeviceEvent;
+import org.onosproject.net.device.DeviceListener;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.host.DefaultHostDescription;
+import org.onosproject.net.host.HostDescription;
+import org.onosproject.net.host.HostProvider;
+import org.onosproject.net.host.HostProviderRegistry;
+import org.onosproject.net.host.HostProviderService;
+import org.onosproject.net.host.HostService;
+import org.onosproject.net.provider.AbstractProvider;
+import org.onosproject.net.provider.ProviderId;
+import org.onosproject.openstackinterface.OpenstackInterfaceService;
+import org.onosproject.openstackinterface.OpenstackNetwork;
+import org.onosproject.openstackinterface.OpenstackPort;
+import org.onosproject.openstackinterface.OpenstackSubnet;
+import org.onosproject.openstacknode.OpenstackNode;
+import org.onosproject.openstacknode.OpenstackNodeEvent;
+import org.onosproject.openstacknode.OpenstackNodeListener;
+import org.onosproject.openstacknode.OpenstackNodeService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Date;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.dhcp.IpAssignment.AssignmentStatus.Option_RangeNotEnforced;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onosproject.net.AnnotationKeys.PORT_NAME;
+import static org.onosproject.openstacknetworking.Constants.*;
+
+@Service
+@Component(immediate = true)
+public final class OpenstackSwitchingHostManager extends AbstractProvider
+        implements HostProvider {
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected CoreService coreService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DeviceService deviceService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected HostProviderRegistry hostProviderRegistry;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DhcpService dhcpService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected HostService hostService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected MastershipService mastershipService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected OpenstackInterfaceService openstackService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected OpenstackNodeService openstackNodeService;
+
+    private final ExecutorService deviceEventExecutor =
+            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackswitching", "device-event"));
+    private final ExecutorService configEventExecutor =
+            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackswitching", "config-event"));
+    private final InternalDeviceListener internalDeviceListener = new InternalDeviceListener();
+    private final InternalOpenstackNodeListener internalNodeListener = new InternalOpenstackNodeListener();
+
+    private HostProviderService hostProvider;
+
+    /**
+     * Creates OpenStack switching host provider.
+     */
+    public OpenstackSwitchingHostManager() {
+        super(new ProviderId("host", SWITCHING_APP_ID));
+    }
+
+    @Activate
+    protected void activate() {
+        coreService.registerApplication(SWITCHING_APP_ID);
+        deviceService.addListener(internalDeviceListener);
+        openstackNodeService.addListener(internalNodeListener);
+        hostProvider = hostProviderRegistry.register(this);
+
+        log.info("Started");
+    }
+
+    @Deactivate
+    protected void deactivate() {
+        hostProviderRegistry.unregister(this);
+        deviceService.removeListener(internalDeviceListener);
+        openstackNodeService.removeListener(internalNodeListener);
+
+        deviceEventExecutor.shutdown();
+        configEventExecutor.shutdown();
+
+        log.info("Stopped");
+    }
+
+    @Override
+    public void triggerProbe(Host host) {
+        // no probe is required
+    }
+
+    private void processPortAdded(Port port) {
+        // TODO check the node state is COMPLETE
+        OpenstackPort osPort = openstackService.port(port);
+        if (osPort == null) {
+            log.warn("Failed to get OpenStack port for {}", port);
+            return;
+        }
+
+        OpenstackNetwork osNet = openstackService.network(osPort.networkId());
+        if (osNet == null) {
+            log.warn("Failed to get OpenStack network {}",
+                    osPort.networkId());
+            return;
+        }
+
+        registerDhcpInfo(osPort);
+        ConnectPoint connectPoint = new ConnectPoint(port.element().id(), port.number());
+        // TODO remove gateway IP from host annotation
+        Map.Entry<String, Ip4Address> fixedIp = osPort.fixedIps().entrySet().stream().findFirst().get();
+
+        // Added CREATE_TIME intentionally to trigger HOST_UPDATED event for the
+        // existing instances.
+        DefaultAnnotations.Builder annotations = DefaultAnnotations.builder()
+                .set(NETWORK_ID, osPort.networkId())
+                .set(SUBNET_ID, fixedIp.getKey())
+                .set(PORT_ID, osPort.id())
+                .set(VXLAN_ID, osNet.segmentId())
+                .set(TENANT_ID, osNet.tenantId())
+                // TODO remove gateway IP from host annotation
+                .set(GATEWAY_IP, fixedIp.getValue().toString())
+                .set(CREATE_TIME, String.valueOf(System.currentTimeMillis()));
+
+        HostDescription hostDesc = new DefaultHostDescription(
+                osPort.macAddress(),
+                VlanId.NONE,
+                new HostLocation(connectPoint, System.currentTimeMillis()),
+                Sets.newHashSet(osPort.fixedIps().values()),
+                annotations.build());
+
+        HostId hostId = HostId.hostId(osPort.macAddress());
+        hostProvider.hostDetected(hostId, hostDesc, false);
+    }
+
+    private void processPortRemoved(Port port) {
+        ConnectPoint connectPoint = new ConnectPoint(port.element().id(), port.number());
+        removeHosts(connectPoint);
+    }
+
+    private void removeHosts(ConnectPoint connectPoint) {
+        hostService.getConnectedHosts(connectPoint).stream()
+                .forEach(host -> {
+                    dhcpService.removeStaticMapping(host.mac());
+                    hostProvider.hostVanished(host.id());
+                });
+    }
+
+    private void registerDhcpInfo(OpenstackPort openstackPort) {
+        checkNotNull(openstackPort);
+        checkArgument(!openstackPort.fixedIps().isEmpty());
+
+        OpenstackSubnet openstackSubnet = openstackService.subnets().stream()
+                .filter(n -> n.networkId().equals(openstackPort.networkId()))
+                .findFirst().orElse(null);
+        if (openstackSubnet == null) {
+            log.warn("Failed to find subnet for {}", openstackPort);
+            return;
+        }
+
+        Ip4Address ipAddress = openstackPort.fixedIps().values().stream().findFirst().get();
+        IpPrefix subnetPrefix = IpPrefix.valueOf(openstackSubnet.cidr());
+        Ip4Address broadcast = Ip4Address.makeMaskedAddress(
+                ipAddress,
+                subnetPrefix.prefixLength());
+
+        // TODO: supports multiple DNS servers
+        Ip4Address domainServer = openstackSubnet.dnsNameservers().isEmpty() ?
+                DNS_SERVER_IP : openstackSubnet.dnsNameservers().get(0);
+
+        IpAssignment ipAssignment = IpAssignment.builder()
+                .ipAddress(ipAddress)
+                .leasePeriod(DHCP_INFINITE_LEASE)
+                .timestamp(new Date())
+                .subnetMask(Ip4Address.makeMaskPrefix(subnetPrefix.prefixLength()))
+                .broadcast(broadcast)
+                .domainServer(domainServer)
+                .assignmentStatus(Option_RangeNotEnforced)
+                .routerAddress(Ip4Address.valueOf(openstackSubnet.gatewayIp()))
+                .build();
+
+        dhcpService.setStaticMapping(openstackPort.macAddress(), ipAssignment);
+    }
+
+    private class InternalDeviceListener implements DeviceListener {
+
+        @Override
+        public void event(DeviceEvent event) {
+            Device device = event.subject();
+            if (!mastershipService.isLocalMaster(device.id())) {
+                // do not allow to proceed without mastership
+                return;
+            }
+
+            Port port = event.port();
+            if (port == null) {
+                return;
+            }
+
+            String portName = port.annotations().value(PORT_NAME);
+            if (Strings.isNullOrEmpty(portName) ||
+                    !portName.startsWith(PORT_NAME_PREFIX_VM)) {
+                // handles VM connected port event only
+                return;
+            }
+
+            switch (event.type()) {
+                case PORT_UPDATED:
+                    if (!event.port().isEnabled()) {
+                        deviceEventExecutor.execute(() -> processPortRemoved(event.port()));
+                    }
+                    break;
+                case PORT_ADDED:
+                    deviceEventExecutor.execute(() -> processPortAdded(event.port()));
+                    break;
+                default:
+                    break;
+            }
+        }
+    }
+
+    private class InternalOpenstackNodeListener implements OpenstackNodeListener {
+
+        @Override
+        public void event(OpenstackNodeEvent event) {
+            OpenstackNode node = event.node();
+            // TODO check leadership of the node and make only the leader process
+
+            switch (event.type()) {
+                case COMPLETE:
+                    log.info("COMPLETE node {} detected", node.hostname());
+
+                    // adds existing VMs running on the complete state node
+                    deviceService.getPorts(node.intBridge()).stream()
+                            .filter(port -> port.annotations().value(PORT_NAME)
+                                    .startsWith(PORT_NAME_PREFIX_VM) &&
+                                    port.isEnabled())
+                            .forEach(port -> {
+                                deviceEventExecutor.execute(() -> processPortAdded(port));
+                                log.info("VM is detected on {}", port);
+                            });
+
+                    // removes stale VMs
+                    hostService.getHosts().forEach(host -> {
+                        if (deviceService.getPort(host.location().deviceId(),
+                                host.location().port()) == null) {
+                            deviceEventExecutor.execute(() -> removeHosts(host.location()));
+                            log.info("Removed stale VM {}", host.location());
+                        }
+                    });
+                    break;
+                case INCOMPLETE:
+                    log.warn("{} is changed to INCOMPLETE state", node);
+                    break;
+                case INIT:
+                case DEVICE_CREATED:
+                default:
+                    break;
+            }
+        }
+    }
+}

 /*
 /*
- * Copyright 2016-present Open Networking Laboratory
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+* Copyright 2016-present Open Networking Laboratory
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
 package org.onosproject.openstacknetworking.switching;
 package org.onosproject.openstacknetworking.switching;
 
 
-import com.google.common.base.Strings;
-import com.google.common.collect.Sets;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
-import org.apache.felix.scr.annotations.Service;
+import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.Ip4Address;
-import org.onlab.packet.IpPrefix;
-import org.onlab.packet.VlanId;
-import org.onosproject.core.CoreService;
-import org.onosproject.dhcp.DhcpService;
-import org.onosproject.dhcp.IpAssignment;
-import org.onosproject.mastership.MastershipService;
-import org.onosproject.net.ConnectPoint;
-import org.onosproject.net.DefaultAnnotations;
-import org.onosproject.net.Device;
+import org.onlab.packet.IpAddress;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.net.DeviceId;
 import org.onosproject.net.Host;
 import org.onosproject.net.Host;
-import org.onosproject.net.HostId;
-import org.onosproject.net.HostLocation;
-import org.onosproject.net.Port;
-import org.onosproject.net.device.DeviceEvent;
-import org.onosproject.net.device.DeviceListener;
+import org.onosproject.net.PortNumber;
 import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.device.DeviceService;
-import org.onosproject.net.host.DefaultHostDescription;
-import org.onosproject.net.host.HostDescription;
-import org.onosproject.net.host.HostProvider;
-import org.onosproject.net.host.HostProviderRegistry;
-import org.onosproject.net.host.HostProviderService;
-import org.onosproject.net.host.HostService;
-import org.onosproject.net.provider.AbstractProvider;
-import org.onosproject.net.provider.ProviderId;
-import org.onosproject.openstackinterface.OpenstackInterfaceService;
-import org.onosproject.openstackinterface.OpenstackNetwork;
-import org.onosproject.openstackinterface.OpenstackPort;
-import org.onosproject.openstackinterface.OpenstackSubnet;
-import org.onosproject.openstacknode.OpenstackNode;
-import org.onosproject.openstacknode.OpenstackNodeEvent;
-import org.onosproject.openstacknode.OpenstackNodeListener;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flowobjective.DefaultForwardingObjective;
+import org.onosproject.net.flowobjective.FlowObjectiveService;
+import org.onosproject.net.flowobjective.ForwardingObjective;
+import org.onosproject.openstacknetworking.AbstractVmHandler;
 import org.onosproject.openstacknode.OpenstackNodeService;
 import org.onosproject.openstacknode.OpenstackNodeService;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;
 
 
-import java.util.Date;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
+import java.util.Objects;
+import java.util.Optional;
 
 
-import static org.onlab.util.Tools.groupedThreads;
-import static org.onosproject.dhcp.IpAssignment.AssignmentStatus.Option_RangeNotEnforced;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.onosproject.net.AnnotationKeys.PORT_NAME;
 import static org.onosproject.openstacknetworking.Constants.*;
 import static org.onosproject.openstacknetworking.Constants.*;
+import static org.onosproject.openstacknetworking.RulePopulatorUtil.buildExtension;
 
 
-@Service
-@Component(immediate = true)
 /**
 /**
- * Populates forwarding rules for VMs created by Openstack.
+ * Populates switching flow rules.
  */
  */
-public final class OpenstackSwitchingManager extends AbstractProvider
-        implements HostProvider {
+@Component(immediate = true)
+public final class OpenstackSwitchingManager extends AbstractVmHandler {
 
 
     private final Logger log = LoggerFactory.getLogger(getClass());
     private final Logger log = LoggerFactory.getLogger(getClass());
 
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected CoreService coreService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected DeviceService deviceService;
     protected DeviceService deviceService;
 
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected HostProviderRegistry hostProviderRegistry;
+    protected FlowObjectiveService flowObjectiveService;
 
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected DhcpService dhcpService;
+    protected OpenstackNodeService nodeService;
 
 
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected HostService hostService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected MastershipService mastershipService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected OpenstackInterfaceService openstackService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected OpenstackNodeService openstackNodeService;
-
-    private final ExecutorService deviceEventExecutor =
-            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackswitching", "device-event"));
-    private final ExecutorService configEventExecutor =
-            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackswitching", "config-event"));
-    private final InternalDeviceListener internalDeviceListener = new InternalDeviceListener();
-    private final InternalOpenstackNodeListener internalNodeListener = new InternalOpenstackNodeListener();
-
-    private HostProviderService hostProvider;
-
-    /**
-     * Creates OpenStack switching host provider.
-     */
-    public OpenstackSwitchingManager() {
-        super(new ProviderId("host", APP_ID));
-    }
+    private ApplicationId appId;
 
 
     @Activate
     @Activate
     protected void activate() {
     protected void activate() {
-        coreService.registerApplication(APP_ID);
-        deviceService.addListener(internalDeviceListener);
-        openstackNodeService.addListener(internalNodeListener);
-        hostProvider = hostProviderRegistry.register(this);
-
-        log.info("Started");
+        super.activate();
+        appId = coreService.registerApplication(SWITCHING_APP_ID);
     }
     }
 
 
     @Deactivate
     @Deactivate
     protected void deactivate() {
     protected void deactivate() {
-        hostProviderRegistry.unregister(this);
-        deviceService.removeListener(internalDeviceListener);
-        openstackNodeService.removeListener(internalNodeListener);
+        super.deactivate();
+    }
 
 
-        deviceEventExecutor.shutdown();
-        configEventExecutor.shutdown();
+    private void populateSwitchingRules(Host host) {
+        populateFlowRulesForTunnelTag(host);
+        populateFlowRulesForTrafficToSameCnode(host);
+        populateFlowRulesForTrafficToDifferentCnode(host);
 
 
-        log.info("Stopped");
+        log.debug("Populated switching rule for {}", host);
     }
     }
 
 
-    @Override
-    public void triggerProbe(Host host) {
-        // no probe is required
+    private void populateFlowRulesForTunnelTag(Host host) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchInPort(host.location().port());
+
+        tBuilder.setTunnelId(Long.valueOf(getVni(host)));
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(TUNNELTAG_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(host.location().deviceId(), fo);
     }
     }
 
 
-    private void processPortAdded(Port port) {
-        // TODO check the node state is COMPLETE
-        OpenstackPort osPort = openstackService.port(port);
-        if (osPort == null) {
-            log.warn("Failed to get OpenStack port for {}", port);
+    private void populateFlowRulesForTrafficToSameCnode(Host host) {
+        //For L2 Switching Case
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(getIp(host).toIpPrefix())
+                .matchTunnelId(Long.valueOf(getVni(host)));
+
+        tBuilder.setOutput(host.location().port());
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(SWITCHING_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(host.location().deviceId(), fo);
+    }
+
+    private void populateFlowRulesForTrafficToDifferentCnode(Host host) {
+        Ip4Address localVmIp = getIp(host);
+        DeviceId localDeviceId = host.location().deviceId();
+        Optional<IpAddress> localDataIp = nodeService.dataIp(localDeviceId);
+
+        if (!localDataIp.isPresent()) {
+            log.debug("Failed to get data IP for device {}",
+                    host.location().deviceId());
             return;
             return;
         }
         }
 
 
-        OpenstackNetwork osNet = openstackService.network(osPort.networkId());
-        if (osNet == null) {
-            log.warn("Failed to get OpenStack network {}",
-                    osPort.networkId());
+        String vni = getVni(host);
+        getVmsInDifferentCnode(host).forEach(remoteVm -> {
+            Optional<IpAddress> remoteDataIp = nodeService.dataIp(remoteVm.location().deviceId());
+            if (remoteDataIp.isPresent()) {
+                setVxLanFlowRule(vni,
+                        localDeviceId,
+                        remoteDataIp.get().getIp4Address(),
+                        getIp(remoteVm));
+
+                setVxLanFlowRule(vni,
+                        remoteVm.location().deviceId(),
+                        localDataIp.get().getIp4Address(),
+                        localVmIp);
+            }
+        });
+    }
+
+    private void setVxLanFlowRule(String vni, DeviceId deviceId, Ip4Address remoteIp,
+                                  Ip4Address vmIp) {
+        Optional<PortNumber> tunnelPort = nodeService.tunnelPort(deviceId);
+        if (!tunnelPort.isPresent()) {
+            log.warn("Failed to get tunnel port from {}", deviceId);
             return;
             return;
         }
         }
 
 
-        registerDhcpInfo(osPort);
-        ConnectPoint connectPoint = new ConnectPoint(port.element().id(), port.number());
-        // TODO remove this and openstackPortInfo
-        String gatewayIp = osNet.subnets().stream().findFirst().get().gatewayIp();
-
-        // Added CREATE_TIME intentionally to trigger HOST_UPDATED event for the
-        // existing instances.
-        DefaultAnnotations.Builder annotations = DefaultAnnotations.builder()
-                .set(NETWORK_ID, osPort.networkId())
-                .set(PORT_ID, osPort.id())
-                .set(VXLAN_ID, osNet.segmentId())
-                .set(TENANT_ID, osNet.tenantId())
-                // TODO remove this and openstackPortInfo
-                .set(GATEWAY_IP, gatewayIp)
-                .set(CREATE_TIME, String.valueOf(System.currentTimeMillis()));
-
-        HostDescription hostDesc = new DefaultHostDescription(
-                osPort.macAddress(),
-                VlanId.NONE,
-                new HostLocation(connectPoint, System.currentTimeMillis()),
-                Sets.newHashSet(osPort.fixedIps().values()),
-                annotations.build());
-
-        HostId hostId = HostId.hostId(osPort.macAddress());
-        hostProvider.hostDetected(hostId, hostDesc, false);
-    }
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(Long.parseLong(vni))
+                .matchIPDst(vmIp.toIpPrefix());
+        tBuilder.extension(buildExtension(deviceService, deviceId, remoteIp), deviceId)
+                .setOutput(tunnelPort.get());
 
 
-    private void processPortRemoved(Port port) {
-        ConnectPoint connectPoint = new ConnectPoint(port.element().id(), port.number());
-        removeHosts(connectPoint);
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(SWITCHING_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(deviceId, fo);
     }
     }
 
 
-    private void removeHosts(ConnectPoint connectPoint) {
-        hostService.getConnectedHosts(connectPoint).stream()
-                .forEach(host -> {
-                    dhcpService.removeStaticMapping(host.mac());
-                    hostProvider.hostVanished(host.id());
-                });
+    private void removeSwitchingRules(Host host) {
+        removeFlowRuleForTunnelTag(host);
+        removeFlowRuleForVMsInSameCnode(host);
+        removeFlowRuleForVMsInDiffrentCnode(host);
+
+        log.debug("Removed switching rule for {}", host);
     }
     }
 
 
-    private void registerDhcpInfo(OpenstackPort openstackPort) {
-        checkNotNull(openstackPort);
-        checkArgument(!openstackPort.fixedIps().isEmpty());
+    private void removeFlowRuleForTunnelTag(Host host) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
 
 
-        OpenstackSubnet openstackSubnet = openstackService.subnets().stream()
-                .filter(n -> n.networkId().equals(openstackPort.networkId()))
-                .findFirst().orElse(null);
-        if (openstackSubnet == null) {
-            log.warn("Failed to find subnet for {}", openstackPort);
-            return;
-        }
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchInPort(host.location().port());
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(TUNNELTAG_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .remove();
+
+        flowObjectiveService.forward(host.location().deviceId(), fo);
+    }
 
 
-        Ip4Address ipAddress = openstackPort.fixedIps().values().stream().findFirst().get();
-        IpPrefix subnetPrefix = IpPrefix.valueOf(openstackSubnet.cidr());
-        Ip4Address broadcast = Ip4Address.makeMaskedAddress(
-                ipAddress,
-                subnetPrefix.prefixLength());
-
-        // TODO: supports multiple DNS servers
-        Ip4Address domainServer = openstackSubnet.dnsNameservers().isEmpty() ?
-                DNS_SERVER_IP : openstackSubnet.dnsNameservers().get(0);
-
-        IpAssignment ipAssignment = IpAssignment.builder()
-                .ipAddress(ipAddress)
-                .leasePeriod(DHCP_INFINITE_LEASE)
-                .timestamp(new Date())
-                .subnetMask(Ip4Address.makeMaskPrefix(subnetPrefix.prefixLength()))
-                .broadcast(broadcast)
-                .domainServer(domainServer)
-                .assignmentStatus(Option_RangeNotEnforced)
-                .routerAddress(Ip4Address.valueOf(openstackSubnet.gatewayIp()))
-                .build();
-
-        dhcpService.setStaticMapping(openstackPort.macAddress(), ipAssignment);
+    private void removeFlowRuleForVMsInSameCnode(Host host) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(getIp(host).toIpPrefix())
+                .matchTunnelId(Long.valueOf(getVni(host)));
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(DefaultTrafficTreatment.builder().build())
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .withPriority(SWITCHING_RULE_PRIORITY)
+                .fromApp(appId)
+                .remove();
+
+        flowObjectiveService.forward(host.location().deviceId(), fo);
     }
     }
 
 
-    private class InternalDeviceListener implements DeviceListener {
+    private void removeFlowRuleForVMsInDiffrentCnode(Host host) {
+        DeviceId deviceId = host.location().deviceId();
+        final boolean anyPortRemainedInSameCnode = hostService.getConnectedHosts(deviceId)
+                .stream()
+                .filter(this::isValidHost)
+                .anyMatch(h -> Objects.equals(getVni(h), getVni(host)));
+
+        getVmsInDifferentCnode(host).forEach(h -> {
+           removeVxLanFlowRule(h.location().deviceId(), getIp(host), getVni(host));
+           if (!anyPortRemainedInSameCnode) {
+               removeVxLanFlowRule(deviceId, getIp(h), getVni(host));
+           }
+       });
+    }
 
 
-        @Override
-        public void event(DeviceEvent event) {
-            Device device = event.subject();
-            if (!mastershipService.isLocalMaster(device.id())) {
-                // do not allow to proceed without mastership
-                return;
-            }
+    private void removeVxLanFlowRule(DeviceId deviceId, Ip4Address vmIp, String vni) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
 
 
-            Port port = event.port();
-            if (port == null) {
-                return;
-            }
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(Long.valueOf(vni))
+                .matchIPDst(vmIp.toIpPrefix());
 
 
-            String portName = port.annotations().value(PORT_NAME);
-            if (Strings.isNullOrEmpty(portName) ||
-                    !portName.startsWith(PORTNAME_PREFIX_VM)) {
-                // handles VM connected port event only
-                return;
-            }
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(DefaultTrafficTreatment.builder().build())
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .withPriority(SWITCHING_RULE_PRIORITY)
+                .fromApp(appId)
+                .remove();
 
 
-            switch (event.type()) {
-                case PORT_UPDATED:
-                    if (!event.port().isEnabled()) {
-                        deviceEventExecutor.execute(() -> processPortRemoved(event.port()));
-                    }
-                    break;
-                case PORT_ADDED:
-                    deviceEventExecutor.execute(() -> processPortAdded(event.port()));
-                    break;
-                default:
-                    break;
-            }
-        }
+        flowObjectiveService.forward(deviceId, fo);
     }
     }
 
 
-    private class InternalOpenstackNodeListener implements OpenstackNodeListener {
-
-        @Override
-        public void event(OpenstackNodeEvent event) {
-            OpenstackNode node = event.node();
-            // TODO check leadership of the node and make only the leader process
-
-            switch (event.type()) {
-                case COMPLETE:
-                    log.info("COMPLETE node {} detected", node.hostname());
-
-                    // adds existing VMs running on the complete state node
-                    deviceService.getPorts(node.intBridge()).stream()
-                            .filter(port -> port.annotations().value(PORT_NAME)
-                                    .startsWith(PORTNAME_PREFIX_VM) &&
-                                    port.isEnabled())
-                            .forEach(port -> {
-                                deviceEventExecutor.execute(() -> processPortAdded(port));
-                                log.info("VM is detected on {}", port);
-                            });
-
-                    // removes stale VMs
-                    hostService.getHosts().forEach(host -> {
-                        if (deviceService.getPort(host.location().deviceId(),
-                                host.location().port()) == null) {
-                            deviceEventExecutor.execute(() -> removeHosts(host.location()));
-                            log.info("Removed stale VM {}", host.location());
-                        }
-                    });
-                    break;
-                case INCOMPLETE:
-                    log.warn("{} is changed to INCOMPLETE state", node);
-                    break;
-                case INIT:
-                case DEVICE_CREATED:
-                default:
-                    break;
-            }
-        }
+    @Override
+    protected void hostDetected(Host host) {
+        populateSwitchingRules(host);
+        log.info("Added new virtual machine to switching service {}", host);
+    }
+
+    @Override
+    protected void hostRemoved(Host host) {
+        removeSwitchingRules(host);
+        log.info("Removed virtual machine from switching service {}", host);
     }
     }
 }
 }

-/*
-* Copyright 2016-present Open Networking Laboratory
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.onosproject.openstacknetworking.switching;
-
-import org.apache.felix.scr.annotations.Activate;
-import org.apache.felix.scr.annotations.Component;
-import org.apache.felix.scr.annotations.Deactivate;
-import org.apache.felix.scr.annotations.Reference;
-import org.apache.felix.scr.annotations.ReferenceCardinality;
-import org.onlab.packet.Ethernet;
-import org.onlab.packet.Ip4Address;
-import org.onlab.packet.IpAddress;
-import org.onosproject.net.Device;
-import org.onosproject.net.DeviceId;
-import org.onosproject.net.Host;
-import org.onosproject.net.PortNumber;
-import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
-import org.onosproject.net.device.DeviceService;
-import org.onosproject.net.flow.DefaultTrafficSelector;
-import org.onosproject.net.flow.DefaultTrafficTreatment;
-import org.onosproject.net.flow.TrafficSelector;
-import org.onosproject.net.flow.TrafficTreatment;
-import org.onosproject.net.flow.instructions.ExtensionTreatment;
-import org.onosproject.net.flow.instructions.ExtensionPropertyException;
-import org.onosproject.net.flowobjective.DefaultForwardingObjective;
-import org.onosproject.net.flowobjective.FlowObjectiveService;
-import org.onosproject.net.flowobjective.ForwardingObjective;
-import org.onosproject.openstacknode.OpenstackNodeService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Objects;
-import java.util.Optional;
-
-import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST;
-import static org.onosproject.openstacknetworking.Constants.*;
-
-/**
- * Populates switching flow rules.
- */
-@Component(immediate = true)
-public final class OpenstackSwitchingRulePopulator extends AbstractVmHandler {
-
-    private final Logger log = LoggerFactory.getLogger(getClass());
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected DeviceService deviceService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected FlowObjectiveService flowObjectiveService;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected OpenstackNodeService nodeService;
-
-
-    private static final String TUNNEL_DST = "tunnelDst";
-
-    @Activate
-    protected void activate() {
-        super.activate();
-    }
-
-    @Deactivate
-    protected void deactivate() {
-        super.deactivate();
-    }
-
-    private void populateSwitchingRules(Host host) {
-        populateFlowRulesForTunnelTag(host);
-        populateFlowRulesForTrafficToSameCnode(host);
-        populateFlowRulesForTrafficToDifferentCnode(host);
-
-        log.debug("Populated switching rule for {}", host);
-    }
-
-    private void populateFlowRulesForTunnelTag(Host host) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchInPort(host.location().port());
-
-        tBuilder.setTunnelId(Long.valueOf(getVni(host)));
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withPriority(TUNNELTAG_RULE_PRIORITY)
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .fromApp(appId)
-                .add();
-
-        flowObjectiveService.forward(host.location().deviceId(), fo);
-    }
-
-    private void populateFlowRulesForTrafficToSameCnode(Host host) {
-        //For L2 Switching Case
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchIPDst(getIp(host).toIpPrefix())
-                .matchTunnelId(Long.valueOf(getVni(host)));
-
-        tBuilder.setOutput(host.location().port());
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withPriority(SWITCHING_RULE_PRIORITY)
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .fromApp(appId)
-                .add();
-
-        flowObjectiveService.forward(host.location().deviceId(), fo);
-    }
-
-    private void populateFlowRulesForTrafficToDifferentCnode(Host host) {
-        Ip4Address localVmIp = getIp(host);
-        DeviceId localDeviceId = host.location().deviceId();
-        Optional<IpAddress> localDataIp = nodeService.dataIp(localDeviceId);
-
-        if (!localDataIp.isPresent()) {
-            log.debug("Failed to get data IP for device {}",
-                    host.location().deviceId());
-            return;
-        }
-
-        String vni = getVni(host);
-        getVmsInDifferentCnode(host).forEach(remoteVm -> {
-            Optional<IpAddress> remoteDataIp = nodeService.dataIp(remoteVm.location().deviceId());
-            if (remoteDataIp.isPresent()) {
-                setVxLanFlowRule(vni,
-                        localDeviceId,
-                        remoteDataIp.get().getIp4Address(),
-                        getIp(remoteVm));
-
-                setVxLanFlowRule(vni,
-                        remoteVm.location().deviceId(),
-                        localDataIp.get().getIp4Address(),
-                        localVmIp);
-            }
-        });
-    }
-
-    private void setVxLanFlowRule(String vni, DeviceId deviceId, Ip4Address remoteIp,
-                                  Ip4Address vmIp) {
-        Optional<PortNumber> tunnelPort = nodeService.tunnelPort(deviceId);
-        if (!tunnelPort.isPresent()) {
-            log.warn("Failed to get tunnel port from {}", deviceId);
-            return;
-        }
-
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchTunnelId(Long.parseLong(vni))
-                .matchIPDst(vmIp.toIpPrefix());
-        tBuilder.extension(buildNiciraExtenstion(deviceId, remoteIp), deviceId)
-                .setOutput(tunnelPort.get());
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withPriority(SWITCHING_RULE_PRIORITY)
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .fromApp(appId)
-                .add();
-
-        flowObjectiveService.forward(deviceId, fo);
-    }
-
-    private void removeSwitchingRules(Host host) {
-        removeFlowRuleForTunnelTag(host);
-        removeFlowRuleForVMsInSameCnode(host);
-        removeFlowRuleForVMsInDiffrentCnode(host);
-
-        log.debug("Removed switching rule for {}", host);
-    }
-
-    private void removeFlowRuleForTunnelTag(Host host) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchInPort(host.location().port());
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withPriority(TUNNELTAG_RULE_PRIORITY)
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .fromApp(appId)
-                .remove();
-
-        flowObjectiveService.forward(host.location().deviceId(), fo);
-    }
-
-    private void removeFlowRuleForVMsInSameCnode(Host host) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchIPDst(getIp(host).toIpPrefix())
-                .matchTunnelId(Long.valueOf(getVni(host)));
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(DefaultTrafficTreatment.builder().build())
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .withPriority(SWITCHING_RULE_PRIORITY)
-                .fromApp(appId)
-                .remove();
-
-        flowObjectiveService.forward(host.location().deviceId(), fo);
-    }
-
-    private void removeFlowRuleForVMsInDiffrentCnode(Host host) {
-        DeviceId deviceId = host.location().deviceId();
-        final boolean anyPortRemainedInSameCnode = hostService.getConnectedHosts(deviceId)
-                .stream()
-                .filter(this::isValidHost)
-                .anyMatch(h -> Objects.equals(getVni(h), getVni(host)));
-
-        getVmsInDifferentCnode(host).forEach(h -> {
-           removeVxLanFlowRule(h.location().deviceId(), getIp(host), getVni(host));
-           if (!anyPortRemainedInSameCnode) {
-               removeVxLanFlowRule(deviceId, getIp(h), getVni(host));
-           }
-       });
-    }
-
-    private void removeVxLanFlowRule(DeviceId deviceId, Ip4Address vmIp, String vni) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchTunnelId(Long.valueOf(vni))
-                .matchIPDst(vmIp.toIpPrefix());
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(DefaultTrafficTreatment.builder().build())
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .withPriority(SWITCHING_RULE_PRIORITY)
-                .fromApp(appId)
-                .remove();
-
-        flowObjectiveService.forward(deviceId, fo);
-    }
-
-    private ExtensionTreatment buildNiciraExtenstion(DeviceId deviceId, Ip4Address remoteIp) {
-        Device device = deviceService.getDevice(deviceId);
-        if (device != null && !device.is(ExtensionTreatmentResolver.class)) {
-            log.error("The extension treatment is not supported");
-            return null;
-        }
-
-        ExtensionTreatmentResolver resolver = device.as(ExtensionTreatmentResolver.class);
-        ExtensionTreatment treatment = resolver.getExtensionInstruction(NICIRA_SET_TUNNEL_DST.type());
-        try {
-            treatment.setPropertyValue(TUNNEL_DST, remoteIp);
-            return treatment;
-        } catch (ExtensionPropertyException e) {
-            log.warn("Failed to get tunnelDst extension treatment for {}", deviceId);
-            return null;
-        }
-    }
-
-    @Override
-    protected void hostDetected(Host host) {
-        populateSwitchingRules(host);
-        log.info("Added new virtual machine to switching service {}", host);
-    }
-
-    @Override
-    protected void hostRemoved(Host host) {
-        removeSwitchingRules(host);
-        log.info("Removed virtual machine from switching service {}", host);
-    }
-}

@@ -126,7 +126,7 @@ public class OpensatckRouterWebResource extends AbstractWebResource {
 
 
             OpenstackRoutingService routingService
             OpenstackRoutingService routingService
                     = getService(OpenstackRoutingService.class);
                     = getService(OpenstackRoutingService.class);
-            routingService.updateRouterInterface(openstackRouterInterface);
+            routingService.addRouterInterface(openstackRouterInterface);
 
 
             log.debug("REST API AddRouterInterface is called from router {} portId: {}, subnetId: {}, tenantId: {}",
             log.debug("REST API AddRouterInterface is called from router {} portId: {}, subnetId: {}, tenantId: {}",
                     openstackRouterInterface.id(), openstackRouterInterface.portId(),
                     openstackRouterInterface.id(), openstackRouterInterface.portId(),
@@ -147,7 +147,7 @@ public class OpensatckRouterWebResource extends AbstractWebResource {
         checkNotNull(id);
         checkNotNull(id);
         OpenstackRoutingService routingService
         OpenstackRoutingService routingService
                 = getService(OpenstackRoutingService.class);
                 = getService(OpenstackRoutingService.class);
-        routingService.deleteRouter(id);
+        routingService.removeRouter(id);
 
 
         log.debug("REST API DELETE routers is called {}", id);
         log.debug("REST API DELETE routers is called {}", id);
         return Response.noContent().build();
         return Response.noContent().build();

@@ -20,7 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.onosproject.openstackinterface.OpenstackFloatingIP;
 import org.onosproject.openstackinterface.OpenstackFloatingIP;
 import org.onosproject.openstackinterface.web.OpenstackFloatingIpCodec;
 import org.onosproject.openstackinterface.web.OpenstackFloatingIpCodec;
-import org.onosproject.openstacknetworking.OpenstackRoutingService;
+import org.onosproject.openstacknetworking.OpenstackFloatingIpService;
 import org.onosproject.rest.AbstractWebResource;
 import org.onosproject.rest.AbstractWebResource;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;
@@ -45,8 +45,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 public class OpenstackFloatingIpWebResource extends AbstractWebResource {
 public class OpenstackFloatingIpWebResource extends AbstractWebResource {
     private final Logger log = LoggerFactory.getLogger(getClass());
     private final Logger log = LoggerFactory.getLogger(getClass());
 
 
-    private static final OpenstackFloatingIpCodec FLOATING_IP_CODEC
-            = new OpenstackFloatingIpCodec();
+    private static final OpenstackFloatingIpCodec FLOATING_IP_CODEC = new OpenstackFloatingIpCodec();
 
 
     /**
     /**
      * Create FloatingIP.
      * Create FloatingIP.
@@ -64,20 +63,15 @@ public class OpenstackFloatingIpWebResource extends AbstractWebResource {
             ObjectMapper mapper = new ObjectMapper();
             ObjectMapper mapper = new ObjectMapper();
             ObjectNode floatingIpNode = (ObjectNode) mapper.readTree(input);
             ObjectNode floatingIpNode = (ObjectNode) mapper.readTree(input);
 
 
-            OpenstackFloatingIP osFloatingIp =
-                    FLOATING_IP_CODEC.decode(floatingIpNode, this);
-
-            OpenstackRoutingService routingService =
-                    getService(OpenstackRoutingService.class);
-
-            routingService.createFloatingIP(osFloatingIp);
+            OpenstackFloatingIP osFloatingIp = FLOATING_IP_CODEC.decode(floatingIpNode, this);
+            OpenstackFloatingIpService floatingIpService =
+                    getService(OpenstackFloatingIpService.class);
+            floatingIpService.createFloatingIp(osFloatingIp);
 
 
             log.debug("REST API CREATE floatingip called");
             log.debug("REST API CREATE floatingip called");
-
             return Response.status(Response.Status.OK).build();
             return Response.status(Response.Status.OK).build();
         } catch (Exception e) {
         } catch (Exception e) {
             log.error("createFloatingIp failed with {}", e.toString());
             log.error("createFloatingIp failed with {}", e.toString());
-
             return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(e.toString())
             return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(e.toString())
                     .build();
                     .build();
         }
         }
@@ -102,20 +96,15 @@ public class OpenstackFloatingIpWebResource extends AbstractWebResource {
             ObjectMapper mapper = new ObjectMapper();
             ObjectMapper mapper = new ObjectMapper();
             ObjectNode floatingIpNode = (ObjectNode) mapper.readTree(input);
             ObjectNode floatingIpNode = (ObjectNode) mapper.readTree(input);
 
 
-            OpenstackFloatingIP osFloatingIp =
-                    FLOATING_IP_CODEC.decode(floatingIpNode, this);
-
-            OpenstackRoutingService routingService =
-                    getService(OpenstackRoutingService.class);
-
-            routingService.updateFloatingIP(osFloatingIp);
+            OpenstackFloatingIP osFloatingIp = FLOATING_IP_CODEC.decode(floatingIpNode, this);
+            OpenstackFloatingIpService floatingIpService =
+                    getService(OpenstackFloatingIpService.class);
+            floatingIpService.updateFloatingIp(osFloatingIp);
 
 
             log.debug("REST API UPDATE floatingip called {}", id);
             log.debug("REST API UPDATE floatingip called {}", id);
-
             return Response.status(Response.Status.OK).build();
             return Response.status(Response.Status.OK).build();
         } catch (Exception e) {
         } catch (Exception e) {
             log.error("updateFloatingIp failed with {}", e.toString());
             log.error("updateFloatingIp failed with {}", e.toString());
-
             return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(e.toString())
             return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(e.toString())
                     .build();
                     .build();
         }
         }
@@ -133,12 +122,11 @@ public class OpenstackFloatingIpWebResource extends AbstractWebResource {
     public Response deleteFloatingIp(@PathParam("id") String id) {
     public Response deleteFloatingIp(@PathParam("id") String id) {
         checkNotNull(id);
         checkNotNull(id);
 
 
-        OpenstackRoutingService routingService =
-                getService(OpenstackRoutingService.class);
-        routingService.deleteFloatingIP(id);
+        OpenstackFloatingIpService floatingIpService =
+                getService(OpenstackFloatingIpService.class);
+        floatingIpService.deleteFloatingIp(id);
 
 
         log.debug("REST API DELETE floatingip is called {}", id);
         log.debug("REST API DELETE floatingip is called {}", id);
-
         return Response.noContent().build();
         return Response.noContent().build();
     }
     }
 
 

@@ -79,7 +79,6 @@ public class OpenstackPortWebResource extends AbstractWebResource {
             sgService.updateSecurityGroup(osPort);
             sgService.updateSecurityGroup(osPort);
 
 
             return Response.status(Response.Status.OK).build();
             return Response.status(Response.Status.OK).build();
-
         } catch (IOException e) {
         } catch (IOException e) {
             log.error("UpdatePort post process failed due to {}", e.getMessage());
             log.error("UpdatePort post process failed due to {}", e.getMessage());
 
 

@@ -28,6 +28,7 @@ import java.util.Optional;
 
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onosproject.openstacknode.Constants.PATCH_INTG_BRIDGE;
 import static org.onosproject.openstacknode.OpenstackNodeEvent.NodeState.INIT;
 import static org.onosproject.openstacknode.OpenstackNodeEvent.NodeState.INIT;
 
 
 /**
 /**
@@ -182,6 +183,20 @@ public final class OpenstackNode {
         return DeviceId.deviceId("ovsdb:" + managementIp.toString());
         return DeviceId.deviceId("ovsdb:" + managementIp.toString());
     }
     }
 
 
+    /**
+     * Returns the name of the port connected to the external network.
+     * It returns valid value only if the node is gateway node.
+     *
+     * @return external port name
+     */
+    public Optional<String> externalPortName() {
+        if (type == NodeType.GATEWAY) {
+            return Optional.of(PATCH_INTG_BRIDGE);
+        } else {
+            return Optional.empty();
+        }
+    }
+
     @Override
     @Override
     public boolean equals(Object obj) {
     public boolean equals(Object obj) {
         if (this == obj) {
         if (this == obj) {