[ONOS-4205] support east-west routing in openstackRouting

- Supports east-west routing traffic - Fixes javadocs Change-Id: I23d9b9497cc2be667fbc9554812c7f5b49c35364

[ONOS-4205] support east-west routing in openstackRouting
- Supports east-west routing traffic - Fixes javadocs Change-Id: I23d9b9497cc2be667fbc9554812c7f5b49c35364
Kyuhwi Choi · Gerrit Code Review
Commit 5e65e2cc138064f16faeb96dc78d1586181f5116 5e65e2cc 1 parent 9cc05f0b
Showing 3 changed files with 292 additions and 55 deletions
apps/openstacknetworking/api/src/main/java/org/onosproject/openstacknetworking/OpenstackRoutingService.java
apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
--- a/apps/openstacknetworking/api/src/main/java/org/onosproject/openstacknetworking/OpenstackRoutingService.java
View file @5e65e2c
+++ b/apps/openstacknetworking/api/src/main/java/org/onosproject/openstacknetworking/OpenstackRoutingService.java
View file @5e65e2c
@@ -20,71 +20,78 @@ import org.onosproject.openstackinterface.OpenstackRouter;
 import org.onosproject.openstackinterface.OpenstackRouterInterface;
 /**
- * The Interface of Openstack Routing.
+ * Supports L3 management REST API for openstack.
  */
 public interface OpenstackRoutingService {
     /**
-     * Stores the Floating IP information created by Openstack.
+     * Stores the floating IP information created by openstack.
      *
-     * @param openstackFloatingIP Floating IP information
+     * @param openstackFloatingIp Floating IP information
      */
-    void createFloatingIP(OpenstackFloatingIP openstackFloatingIP);
+    void createFloatingIP(OpenstackFloatingIP openstackFloatingIp);
     /**
-     * Updates flow rules corresponding to the Floating IP information updated by Openstack.
+     * Updates flow rules corresponding to the floating IP information updated by openstack.
      *
-     * @param openstackFloatingIP Floating IP information
+     * @param openstackFloatingIp Floating IP information
      */
-    void updateFloatingIP(OpenstackFloatingIP openstackFloatingIP);
+    void updateFloatingIP(OpenstackFloatingIP openstackFloatingIp);
     /**
-     * Removes flow rules corresponding to Floating IP information removed by Openstack.
+     * Removes flow rules corresponding to floating IP information removed by openstack.
      *
      * @param id Deleted Floating IP`s ID
      */
     void deleteFloatingIP(String id);
     /**
-     * Stores the router information created by Openstack.
+     * Stores the router information created by openstack.
      *
-     * @param openstackRouter Floating IP information
+     * @param openstackRouter Router information
      */
     void createRouter(OpenstackRouter openstackRouter);
     /**
-     * Updates flow rules corresponding to the router information updated by Openstack.
+     * Updates flow rules corresponding to the router information updated by openstack.
      *
      * @param openstackRouter Router information
      */
     void updateRouter(OpenstackRouter openstackRouter);
     /**
-     * Removes flow rules corresponding to the router information removed by Openstack.
+     * Removes flow rules corresponding to the router information removed by openstack.
      *
      * @param id Deleted router`s ID
      */
     void deleteRouter(String id);
     /**
-     * Updates flow rules corresponding to the router information updated by Openstack.
+     * Updates flow rules corresponding to the router information updated by openstack.
      *
-     * @param openstackRouterInterface Router information
+     * @param openstackRouterInterface Router interface information
      */
     void updateRouterInterface(OpenstackRouterInterface openstackRouterInterface);
     /**
-     * Removes flow rules corresponding to the router information removed by Openstack.
+     * Removes flow rules corresponding to the router information removed by openstack.
      *
-     * @param openstackRouterInterface Router information
+     * @param openstackRouterInterface Router interface information
      */
     void removeRouterInterface(OpenstackRouterInterface openstackRouterInterface);
     /**
-     * Checks floatingIp deassociation when corresponding deleted vm.
+     * Checks floatingIp disassociation when corresponding deleted vm.
      *
      * @param portId Deleted vm
      * @param portInfo stored information about deleted vm
      */
     void checkDisassociatedFloatingIp(String portId, OpenstackPortInfo portInfo);
+
+    /**
+     * Returns network id for routerInterface.
+     *
+     * @param portId routerInterface`s port id
+     */
+    String networkIdForRouterInterface(String portId);
 }
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
View file @5e65e2c
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
View file @5e65e2c
@@ -37,6 +37,8 @@ import org.onosproject.net.config.NetworkConfigEvent;
 import org.onosproject.net.config.NetworkConfigListener;
 import org.onosproject.net.config.NetworkConfigRegistry;
 import org.onosproject.net.config.NetworkConfigService;
+import org.onosproject.net.device.DeviceEvent;
+import org.onosproject.net.device.DeviceListener;
 import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.driver.DriverService;
 import org.onosproject.net.flowobjective.FlowObjectiveService;
@@ -115,12 +117,15 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
     private ApplicationId appId;
     private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
     private ConsistentMap<String, OpenstackFloatingIP> floatingIpMap; // Map<FloatingIp`s Id, FloatingIp object>
+    // Map<RouterInterface`s portId, Corresponded port`s network id>
+    private ConsistentMap<String, String> routerInterfaceMap;
     private static final String APP_ID = "org.onosproject.openstackrouting";
     private static final String PORT_NAME = "portName";
     private static final String PORTNAME_PREFIX_VM = "tap";
     private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
     private static final String FLOATING_IP_MAP_NAME = "openstackrouting-floatingip";
-    private static final String TP_PORT_MAP_NAME = "openstackrouting-portnum";
+    private static final String TP_PORT_MAP_NAME = "openstackrouting-tpportnum";
+    private static final String ROUTER_INTERFACE_MAP_NAME = "openstackrouting-routerinterface";
     private static final String COLON = ":";
     private static final int PNAT_PORT_EXPIRE_TIME = 1200 * 1000;
     private static final int TP_PORT_MINIMUM_NUM = 1024;
@@ -152,7 +157,13 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
             .register(Integer.class)
             .register(String.class);
+    private static final KryoNamespace.Builder ROUTER_INTERFACE_SERIALIZER = KryoNamespace.newBuilder()
+            .register(KryoNamespaces.API)
+            .register(KryoNamespaces.MISC)
+            .register(String.class);
+
     private InternalPacketProcessor internalPacketProcessor = new InternalPacketProcessor();
+    private InternalDeviceListener internalDeviceListener = new InternalDeviceListener();
     private ExecutorService l3EventExecutorService =
             Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "L3-event"));
     private ExecutorService icmpEventExecutorService =
@@ -170,6 +181,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
         packetService.addProcessor(internalPacketProcessor, PacketProcessor.director(1));
         configRegistry.registerConfigFactory(configFactory);
         configService.addListener(configListener);
+        deviceService.addListener(internalDeviceListener);
         floatingIpMap = storageService.<String, OpenstackFloatingIP>consistentMapBuilder()
                 .withSerializer(Serializer.using(FLOATING_IP_SERIALIZER.build()))
@@ -181,52 +193,59 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
                 .withName(TP_PORT_MAP_NAME)
                 .withApplicationId(appId)
                 .build();
+        routerInterfaceMap = storageService.<String, String>consistentMapBuilder()
+                .withSerializer(Serializer.using(ROUTER_INTERFACE_SERIALIZER.build()))
+                .withName(ROUTER_INTERFACE_MAP_NAME)
+                .withApplicationId(appId)
+                .build();
         readConfiguration();
-        log.info("onos-openstackrouting started");
+        log.info("started");
     }
     @Deactivate
     protected void deactivate() {
         packetService.removeProcessor(internalPacketProcessor);
+        deviceService.removeListener(internalDeviceListener);
         l3EventExecutorService.shutdown();
         icmpEventExecutorService.shutdown();
         arpEventExecutorService.shutdown();
         floatingIpMap.clear();
         tpPortNumMap.clear();
+        routerInterfaceMap.clear();
-        log.info("onos-openstackrouting stopped");
+        log.info("stopped");
     }
     @Override
-    public void createFloatingIP(OpenstackFloatingIP openstackFloatingIP) {
+    public void createFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
-        floatingIpMap.put(openstackFloatingIP.id(), openstackFloatingIP);
+        floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
     }
     @Override
-    public void updateFloatingIP(OpenstackFloatingIP openstackFloatingIP) {
+    public void updateFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
-        if (!floatingIpMap.containsKey(openstackFloatingIP.id())) {
+        if (!floatingIpMap.containsKey(openstackFloatingIp.id())) {
-            log.warn("There`s no information about {} in FloatingIpMap", openstackFloatingIP.id());
+            log.warn("There`s no information about {} in FloatingIpMap", openstackFloatingIp.id());
             return;
         }
-        if (openstackFloatingIP.portId() == null || openstackFloatingIP.portId().equals("null")) {
+        if (openstackFloatingIp.portId() == null || openstackFloatingIp.portId().equals("null")) {
-            OpenstackFloatingIP floatingIP = floatingIpMap.get(openstackFloatingIP.id()).value();
+            OpenstackFloatingIP floatingIp = floatingIpMap.get(openstackFloatingIp.id()).value();
             OpenstackPortInfo portInfo = openstackSwitchingService.openstackPortInfo()
-                    .get(PORTNAME_PREFIX_VM.concat(floatingIP.portId().substring(0, 11)));
+                    .get(PORTNAME_PREFIX_VM.concat(floatingIp.portId().substring(0, 11)));
             if (portInfo == null) {
-                log.warn("There`s no portInfo information about portId {}", floatingIP.portId());
+                log.warn("There`s no portInfo information about portId {}", floatingIp.portId());
                 return;
             }
             l3EventExecutorService.execute(
-                    new OpenstackFloatingIPHandler(rulePopulator, floatingIP, false, portInfo));
+                    new OpenstackFloatingIPHandler(rulePopulator, floatingIp, false, portInfo));
-            floatingIpMap.replace(floatingIP.id(), openstackFloatingIP);
+            floatingIpMap.replace(floatingIp.id(), openstackFloatingIp);
         } else {
-            floatingIpMap.put(openstackFloatingIP.id(), openstackFloatingIP);
+            floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
             l3EventExecutorService.execute(
-                    new OpenstackFloatingIPHandler(rulePopulator, openstackFloatingIP, true, null));
+                    new OpenstackFloatingIPHandler(rulePopulator, openstackFloatingIp, true, null));
         }
     }
@@ -280,11 +299,69 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
         List<OpenstackRouterInterface> routerInterfaces = Lists.newArrayList();
         routerInterfaces.add(routerInterface);
         checkExternalConnection(getOpenstackRouter(routerInterface.id()), routerInterfaces);
+        setL3Connection(getOpenstackRouter(routerInterface.id()), null);
+        routerInterfaceMap.put(routerInterface.portId(), openstackService.port(routerInterface.portId()).networkId());
+    }
+
+    private void setL3Connection(OpenstackRouter openstackRouter, OpenstackPort openstackPort) {
+        Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(openstackRouter);
+        if (interfaceList.size() < 2) {
+            return;
+        }
+        if (openstackPort == null) {
+            interfaceList.forEach(i -> {
+                OpenstackPort interfacePort = openstackService.port(i.portId());
+                openstackService.ports()
+                        .stream()
+                        .filter(p -> p.networkId().equals(interfacePort.networkId())
+                                && !p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
+                        .forEach(p -> rulePopulator.populateL3Rules(p,
+                                getL3ConnectionList(p.networkId(), interfaceList)));
+            });
+        } else {
+            rulePopulator.populateL3Rules(openstackPort, getL3ConnectionList(openstackPort.networkId(), interfaceList));
+        }
+
+    }
+
+    private List<OpenstackRouterInterface> getL3ConnectionList(String networkId,
+                                                               Collection<OpenstackRouterInterface> interfaceList) {
+        List<OpenstackRouterInterface> targetList = Lists.newArrayList();
+        interfaceList.forEach(i -> {
+            OpenstackPort port = openstackService.port(i.portId());
+            if (!port.networkId().equals(networkId)) {
+                targetList.add(i);
+            }
+        });
+        return targetList;
     }
     @Override
     public void removeRouterInterface(OpenstackRouterInterface routerInterface) {
+        OpenstackRouter router = openstackService.router(routerInterface.id());
+        Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(router);
+        if (interfaceList.size() == 1) {
+            List<OpenstackRouterInterface> newList = Lists.newArrayList();
+            newList.add(routerInterface);
+            interfaceList.forEach(i -> removeL3RulesForRouterInterface(i, router, newList));
+        }
+        removeL3RulesForRouterInterface(routerInterface, router, null);
         rulePopulator.removeExternalRules(routerInterface);
+        routerInterfaceMap.remove(routerInterface.portId());
+    }
+
+    private void removeL3RulesForRouterInterface(OpenstackRouterInterface routerInterface, OpenstackRouter router,
+                                                 List<OpenstackRouterInterface> newList) {
+        openstackService.ports(routerInterfaceMap.get(routerInterface.portId()).value()).forEach(p -> {
+                    Ip4Address vmIp = (Ip4Address) p.fixedIps().values().toArray()[0];
+                    if (newList == null) {
+                        rulePopulator.removeL3Rules(vmIp,
+                                getL3ConnectionList(p.networkId(), getOpenstackRouterInterface(router)));
+                    } else {
+                        rulePopulator.removeL3Rules(vmIp, newList);
+                    }
+                }
+        );
     }
     @Override
@@ -313,6 +390,11 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
         }
     }
+    @Override
+    public String networkIdForRouterInterface(String portId) {
+        return routerInterfaceMap.get(portId).value();
+    }
+
     private Collection<OpenstackFloatingIP> associatedFloatingIps() {
         List<OpenstackFloatingIP> fIps = Lists.newArrayList();
         floatingIpMap.values()
@@ -327,10 +409,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
                         openstackService.ports()
                                 .stream()
                                 .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
-                                .forEach(p -> {
+                                .forEach(p -> updateRouterInterface(portToRouterInterface(p)))
-                                    OpenstackRouterInterface routerInterface = portToRouterInterface(p);
-                                    updateRouterInterface(routerInterface);
-                                })
         );
     }
@@ -382,14 +461,13 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
                         int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
                         Optional<Port> port =
                                 getExternalPort(pkt.receivedFrom().deviceId(), config.gatewayExternalInterfaceName());
-
+                        if (port.isPresent()) {
-                        if (!port.isPresent()) {
-                            log.warn("There`s no external interface");
-                        } else {
                             OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
                                     Ip4Address.valueOf(iPacket.getSourceAddress()));
                             l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
                                     portNum, openstackPort, port.get(), config));
+                        } else {
+                            log.warn("There`s no external interface");
                         }
                         break;
                 }
@@ -471,22 +549,20 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
         List<OpenstackRouterInterface> interfaces = Lists.newArrayList();
         openstackService.ports()
                 .stream()
-                .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
+                .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
-                .filter(p -> p.deviceId().equals(router.id()))
+                        && p.deviceId().equals(router.id()))
-                .forEach(p -> {
+                .forEach(p -> interfaces.add(portToRouterInterface(p)));
-                    interfaces.add(portToRouterInterface(p));
-                });
         return interfaces;
     }
     private OpenstackRouter getOpenstackRouter(String id) {
         return openstackService.routers().stream().filter(r ->
-                r.id().equals(id)).findAny().orElse(null);
+                r.id().equals(id)).iterator().next();
     }
     private OpenstackPort getOpenstackPort(MacAddress sourceMac, Ip4Address ip4Address) {
         OpenstackPort openstackPort = openstackService.ports().stream()
-                .filter(p -> p.macAddress().equals(sourceMac)).findFirst().orElse(null);
+                .filter(p -> p.macAddress().equals(sourceMac)).iterator().next();
         return openstackPort.fixedIps().values().stream().filter(ip ->
                 ip.equals(ip4Address)).count() > 0 ? openstackPort : null;
     }
@@ -534,4 +610,38 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
             }
         }
     }
+
+    private class InternalDeviceListener implements DeviceListener {
+
+        @Override
+        public void event(DeviceEvent deviceEvent) {
+            if (deviceEvent.type() == DeviceEvent.Type.PORT_UPDATED) {
+                Port port = deviceEvent.port();
+                OpenstackPortInfo portInfo = openstackSwitchingService.openstackPortInfo()
+                        .get(port.annotations().value(PORT_NAME));
+                OpenstackPort openstackPort = openstackService.port(port);
+                OpenstackPort interfacePort = openstackService.ports()
+                        .stream()
+                        .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
+                                && p.networkId().equals(openstackPort.networkId()))
+                        .findAny()
+                        .orElse(null);
+                if (portInfo == null && openstackPort == null) {
+                    log.warn("As delete event timing issue between routing and switching, Can`t delete L3 rules");
+                    return;
+                }
+                if ((port.isEnabled()) && (interfacePort != null)) {
+                    OpenstackRouterInterface routerInterface = portToRouterInterface(interfacePort);
+                    l3EventExecutorService.execute(() ->
+                            setL3Connection(getOpenstackRouter(routerInterface.id()), openstackPort));
+                } else if (interfacePort != null) {
+                    OpenstackRouterInterface routerInterface = portToRouterInterface(interfacePort);
+                    l3EventExecutorService.execute(() -> rulePopulator.removeL3Rules(portInfo.ip(),
+                            getL3ConnectionList(portInfo.networkId(),
+                                    getOpenstackRouterInterface(getOpenstackRouter(routerInterface.id())))));
+                }
+            }
+        }
+    }
+
 }
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
View file @5e65e2c
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
View file @5e65e2c
@@ -54,12 +54,15 @@ import org.onosproject.openstackinterface.OpenstackSubnet;
 import org.onosproject.openstackinterface.OpenstackFloatingIP;
 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
 import org.onosproject.openstacknetworking.OpenstackPortInfo;
+import org.onosproject.openstacknetworking.OpenstackRoutingService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import java.util.List;
 import java.util.stream.StreamSupport;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onlab.osgi.DefaultServiceDirectory.getService;
 /**
  * Populates Routing Flow Rules.
@@ -99,11 +102,11 @@ public class OpenstackRoutingRulePopulator {
      * The constructor of openstackRoutingRulePopulator.
      *
      * @param appId Caller`s appId
-     * @param openstackService OpenstackNetworkingService
+     * @param openstackService Opestack REST request handler
      * @param flowObjectiveService FlowObjectiveService
      * @param deviceService DeviceService
      * @param driverService DriverService
-     * @param config OpenstackRoutingConfig
+     * @param config Configuration for openstack environment
      */
     public OpenstackRoutingRulePopulator(ApplicationId appId, OpenstackInterfaceService openstackService,
                                          FlowObjectiveService flowObjectiveService, DeviceService deviceService,
@@ -187,6 +190,7 @@ public class OpenstackRoutingRulePopulator {
                 .add();
         flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo);
+
     }
     private Port getPortOfExternalInterface() {
@@ -252,8 +256,8 @@ public class OpenstackRoutingRulePopulator {
         String openstackPortName = PORTNAME_PREFIX_VM + openstackPort.id().substring(0, 11);
         Device device = StreamSupport.stream(deviceService.getDevices().spliterator(), false)
                 .filter(d -> findPortinDevice(d, openstackPortName))
-                .findAny()
+                .iterator()
-                .orElse(null);
+                .next();
         checkNotNull(device, DEVICENOTNULL);
         return device;
     }
@@ -268,7 +272,7 @@ public class OpenstackRoutingRulePopulator {
     }
     /**
-     * Builds NiciraExtension for tagging remoteIp of vxlan.
+     * Builds Nicira extension for tagging remoteIp of vxlan.
      *
      * @param id Device Id of vxlan source device
      * @param hostIp Remote Ip of vxlan destination device
@@ -401,8 +405,8 @@ public class OpenstackRoutingRulePopulator {
         StreamSupport.stream(deviceService.getDevices().spliterator(), false)
                 .forEach(d -> {
                     ForwardingObjective.Flag flag = checkGatewayNode(d.id()) ?
-                            ForwardingObjective.Flag.VERSATILE : ForwardingObjective.Flag.SPECIFIC;
+                            ForwardingObjective.Flag.VERSATILE :
-
+                            ForwardingObjective.Flag.SPECIFIC;
                     removeRule(d.id(), sBuilder, flag, ROUTING_RULE_PRIORITY);
                 });
@@ -514,4 +518,120 @@ public class OpenstackRoutingRulePopulator {
         removeRule(getGatewayNode().id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
     }
+    /**
+     * Populates L3 rules for east to west traffic.
+     *
+     * @param p target VM
+     * @param targetList target openstackRouterInterfaces
+     */
+    public void populateL3Rules(OpenstackPort p, List<OpenstackRouterInterface> targetList) {
+        Device device = getDevicefromOpenstackPort(p);
+        Port port = getPortFromOpenstackPort(device, p);
+        Ip4Address vmIp = (Ip4Address) p.fixedIps().values().iterator().next();
+
+        if (port == null) {
+            return;
+        }
+
+        targetList.forEach(routerInterface -> {
+            OpenstackPort openstackPort = openstackService.port(routerInterface.portId());
+            long vni = getVni(openstackPort.networkId());
+
+            if (vmIp == null) {
+                return;
+            }
+
+            populateL3RulestoSameNode(vmIp, p, port, device, vni);
+
+            deviceService.getAvailableDevices().forEach(d -> {
+                if (!d.equals(device) && !d.equals(getGatewayNode())) {
+                    populateL3RulestoDifferentNode(vmIp, vni, d.id(), getHostIpfromOpenstackPort(p));
+                }
+            });
+
+        });
+    }
+
+    private void populateL3RulestoDifferentNode(Ip4Address vmIp, long vni, DeviceId deviceId, Ip4Address hostIp) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(vni)
+                .matchIPDst(vmIp.toIpPrefix());
+        tBuilder.extension(buildNiciraExtenstion(deviceId, hostIp), deviceId)
+                .setOutput(getTunnelPort(deviceId));
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(ROUTING_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(deviceId, fo);
+    }
+
+    private void populateL3RulestoSameNode(Ip4Address vmIp, OpenstackPort p, Port port, Device device, long vni) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(vmIp.toIpPrefix())
+                .matchTunnelId(vni);
+
+        tBuilder.setEthDst(p.macAddress())
+                .setOutput(port.number());
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(ROUTING_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(device.id(), fo);
+    }
+
+    private Port getPortFromOpenstackPort(Device device, OpenstackPort p) {
+        String openstackPortName = PORTNAME_PREFIX_VM + p.id().substring(0, 11);
+        return  deviceService.getPorts(device.id())
+                .stream()
+                .filter(pt -> pt.annotations().value(PORTNAME).equals(openstackPortName))
+                .findAny()
+                .orElse(null);
+    }
+
+    /**
+     * Removes L3 rules for routerInterface events.
+     *
+     * @param vmIp Corresponding Vm ip
+     * @param routerInterfaces Corresponding routerInterfaces
+     */
+    public void removeL3Rules(Ip4Address vmIp, List<OpenstackRouterInterface> routerInterfaces) {
+        if (vmIp == null) {
+            return;
+        }
+
+        OpenstackRoutingService routingService = getService(OpenstackRoutingService.class);
+
+        deviceService.getAvailableDevices().forEach(d -> {
+            if (!d.equals(getGatewayNode())) {
+                routerInterfaces.forEach(routerInterface -> {
+                    String networkId = routingService.networkIdForRouterInterface(routerInterface.portId());
+                    long vni = getVni(networkId);
+
+                    TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+
+                    sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                            .matchIPDst(vmIp.toIpPrefix())
+                            .matchTunnelId(vni);
+
+                    removeRule(d.id(), sBuilder, ForwardingObjective.Flag.SPECIFIC, ROUTING_RULE_PRIORITY);
+                });
+            }
+        });
+    }
 }