[ONOS-3696] Remove flow rules when a VM is terminated

- ProcessPortAdded method name is changed to ProcessPortUpdated - PortNumber is removed in OpenstackPortInfo class - Checks doNotPushFlows in OpenstackSwitchingRulePopulator - etc - rebased Change-Id: I215892102669799af0fd8c8ac8ea377ab7be2aad

[ONOS-3696] Remove flow rules when a VM is terminated
- ProcessPortAdded method name is changed to ProcessPortUpdated - PortNumber is removed in OpenstackPortInfo class - Checks doNotPushFlows in OpenstackSwitchingRulePopulator - etc - rebased Change-Id: I215892102669799af0fd8c8ac8ea377ab7be2aad
Daniel Park · Gerrit Code Review
Commit 5df6518dd26d1305947cdd216d290d19500287be 5df6518d 1 parent 69fc2ffb
Showing 3 changed files with 341 additions and 228 deletions
apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackPortInfo.java
apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingManager.java
apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingRulePopulator.java
--- a/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackPortInfo.java 0 → 100644
View file @5df6518
+++ b/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackPortInfo.java 0 → 100644
View file @5df6518
+/*
+ * Copyright 2014-2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstackswitching;
+
+import org.onlab.packet.Ip4Address;
+import org.onosproject.net.DeviceId;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Contains OpenstackPort Information.
+ */
+public final class OpenstackPortInfo {
+
+    private final Ip4Address hostIp;
+    private final DeviceId deviceId;
+    private final long vni;
+
+    public OpenstackPortInfo(Ip4Address hostIp, DeviceId deviceId,
+                             long vni) {
+        this.hostIp = hostIp;
+        this.deviceId = deviceId;
+        this.vni = vni;
+    }
+
+    public Ip4Address ip() {
+        return hostIp;
+    }
+
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    public long vni() {
+        return vni;
+    }
+
+    public static OpenstackPortInfo.Builder builder() {
+        return new Builder();
+    }
+
+    public static final class Builder {
+        private Ip4Address hostIp;
+        private DeviceId deviceId;
+        private long vni;
+
+        public Builder setHostIp(Ip4Address hostIp) {
+            this.hostIp = checkNotNull(hostIp, "hostIp cannot be null");
+            return this;
+        }
+
+        public Builder setDeviceId(DeviceId deviceId) {
+            this.deviceId = checkNotNull(deviceId, "deviceId cannot be null");
+            return this;
+        }
+
+        public Builder setVNI(long vni) {
+            this.vni = checkNotNull(vni, "vni cannot be null");
+            return this;
+        }
+        public OpenstackPortInfo build() {
+            return new OpenstackPortInfo(this);
+        }
+    }
+
+    private OpenstackPortInfo(Builder builder) {
+        hostIp = builder.hostIp;
+        deviceId = builder.deviceId;
+        vni = builder.vni;
+    }
+}
--- a/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingManager.java
View file @5df6518
+++ b/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingManager.java
View file @5df6518
@@ -17,6 +17,7 @@ package org.onosproject.openstackswitching;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -25,13 +26,11 @@ import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.Service;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ip4Address;
-import org.onlab.packet.IpAddress;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.CoreService;
 import org.onosproject.dhcp.DhcpService;
 import org.onosproject.event.AbstractEvent;
 import org.onosproject.net.Device;
-import org.onosproject.net.DeviceId;
 import org.onosproject.net.Host;
 import org.onosproject.net.Port;
 import org.onosproject.net.config.ConfigFactory;
@@ -42,12 +41,6 @@ import org.onosproject.net.device.DeviceEvent;
 import org.onosproject.net.device.DeviceListener;
 import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.driver.DriverService;
-import org.onosproject.net.flow.FlowEntry;
-import org.onosproject.net.flow.FlowRuleService;
-import org.onosproject.net.flow.criteria.Criterion;
-import org.onosproject.net.flow.criteria.IPCriterion;
-import org.onosproject.net.flow.instructions.Instruction;
-import org.onosproject.net.flow.instructions.L2ModificationInstruction;
 import org.onosproject.net.flowobjective.FlowObjectiveService;
 import org.onosproject.net.host.HostEvent;
 import org.onosproject.net.host.HostListener;
@@ -60,6 +53,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import java.util.List;
 import java.util.Collection;
+import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
@@ -104,9 +98,6 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected DriverService driverService;
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected FlowRuleService flowRuleService;
-
     private ApplicationId appId;
     private boolean doNotPushFlows;
     private Ip4Address neutronServer;
@@ -136,6 +127,9 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
             }
     );
+
+    private Map<String, OpenstackPortInfo> openstackPortInfoMap = Maps.newHashMap();
+
     @Activate
     protected void activate() {
         appId = coreService
@@ -258,13 +252,39 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
         log.debug("device {} is added", device.id());
     }
-    private void processPortAdded(Device device, Port port) {
+    private void processPortUpdated(Device device, Port port) {
-        if (!port.annotations().value("portName").equals("vxlan")
+        if (!port.annotations().value("portName").equals("vxlan") && !doNotPushFlows) {
-            && port.isEnabled() && !doNotPushFlows) {
+            if (port.isEnabled()) {
                 OpenstackSwitchingRulePopulator rulePopulator =
                         new OpenstackSwitchingRulePopulator(appId, flowObjectiveService,
                                 deviceService, restHandler, driverService);
                 rulePopulator.populateSwitchingRules(device, port);
+
+                OpenstackPort openstackPort = port(port);
+
+                long vni = Long.parseLong(restHandler.getNetworks().stream()
+                        .filter(n -> n.id().equals(openstackPort.networkId()))
+                        .findAny().orElse(null).segmentId());
+
+                OpenstackPortInfo.Builder portBuilder = OpenstackPortInfo.builder()
+                        .setDeviceId(device.id())
+                        .setHostIp((Ip4Address) openstackPort.fixedIps().values().stream().findFirst().orElse(null))
+                        .setVNI(vni);
+
+                openstackPortInfoMap.putIfAbsent(port.annotations().value("portName"),
+                        portBuilder.build());
+            }
+
+            //In case portupdate event is driven by vm shutoff from openstack
+            if (!port.isEnabled() && openstackPortInfoMap.containsKey(port.annotations().value("portName"))) {
+                log.debug("Flowrules according to the port {} were removed", port.number().toString());
+                OpenstackSwitchingRulePopulator rulePopulator =
+                        new OpenstackSwitchingRulePopulator(appId, flowObjectiveService,
+                                deviceService, restHandler, driverService);
+                openstackPortInfoMap.get(port.annotations().value("portName"));
+                rulePopulator.removeSwitchingRules(port, openstackPortInfoMap);
+                openstackPortInfoMap.remove(port.annotations().value("portName"));
+            }
         }
     }
@@ -300,46 +320,6 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
     private void processHostRemoved(Host host) {
         log.debug("host {} was removed", host.toString());
-
-        try {
-            if (!doNotPushFlows) {
-                IpAddress hostIp = host.ipAddresses().stream().
-                        filter(ip -> ip.isIp4()).findAny().orElse(null);
-                OpenstackSwitchingRulePopulator rulePopulator =
-                        new OpenstackSwitchingRulePopulator(appId, flowObjectiveService,
-                                deviceService, restHandler, driverService);
-                rulePopulator.removeSwitchingRules(host.location().deviceId(),
-                        hostIp.getIp4Address());
-            }
-
-            dhcpService.removeStaticMapping(host.mac());
-        } catch (NoSuchElementException e) {
-            log.error("No IP address is assigned.");
-        }
-    }
-
-    private long getVniFromFlowRules(DeviceId deviceId, Ip4Address hostIp) {
-
-        for (FlowEntry flowEntry: flowRuleService.getFlowEntries(deviceId)) {
-            Criterion c = flowEntry.selector().getCriterion(Criterion.Type.IPV4_DST);
-            if (c != null) {
-                IPCriterion destIpCriterion = (IPCriterion) c;
-                if (destIpCriterion.ip().getIp4Prefix().address().equals(hostIp)) {
-                    for (Instruction i : flowEntry.treatment().immediate()) {
-                        if (i.type().equals(Instruction.Type.L2MODIFICATION)) {
-                            L2ModificationInstruction l2m = (L2ModificationInstruction) i;
-                            if (l2m.subtype().equals(L2ModificationInstruction.L2SubType.TUNNEL_ID)) {
-                                L2ModificationInstruction.ModTunnelIdInstruction setTunnelInstr =
-                                        (L2ModificationInstruction.ModTunnelIdInstruction) l2m;
-                                return setTunnelInstr.tunnelId();
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        return 0;
     }
     private void registerDhcpInfo(OpenstackPort openstackPort) {
@@ -350,7 +330,7 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
         Ip4Address domainServer;
         OpenstackSubnet openstackSubnet;
-        ip4Address = (Ip4Address) openstackPort.fixedIps().values().toArray()[0];
+        ip4Address = (Ip4Address) openstackPort.fixedIps().values().stream().findFirst().orElse(null);
         openstackSubnet = restHandler.getSubnets().stream()
                 .filter(n -> n.networkId().equals(openstackPort.networkId()))
@@ -442,7 +422,7 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
                     case DEVICE_UPDATED:
                         Port port = (Port) deviceEvent.subject();
                         if (port.isEnabled()) {
-                            processPortAdded((Device) deviceEvent.subject(), deviceEvent.port());
+                            processPortUpdated((Device) deviceEvent.subject(), deviceEvent.port());
                         }
                         break;
                     case DEVICE_AVAILABILITY_CHANGED:
@@ -452,10 +432,10 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
                         }
                         break;
                     case PORT_ADDED:
-                        processPortAdded((Device) deviceEvent.subject(), deviceEvent.port());
+                        processPortUpdated((Device) deviceEvent.subject(), deviceEvent.port());
                         break;
                     case PORT_UPDATED:
-                        processPortAdded((Device) deviceEvent.subject(), deviceEvent.port());
+                        processPortUpdated((Device) deviceEvent.subject(), deviceEvent.port());
                         break;
                     case PORT_REMOVED:
                         processPortRemoved((Device) deviceEvent.subject(), deviceEvent.port());
@@ -502,19 +482,4 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
         }
     }
-
-    private final class PortInfo {
-        DeviceId deviceId;
-        String portName;
-        Ip4Address fixedIp;
-        Ip4Address hostIp;
-
-        private PortInfo(DeviceId deviceId, String portName, Ip4Address fixedIp,
-                         Ip4Address hostIp) {
-            this.deviceId = deviceId;
-            this.portName = portName;
-            this.fixedIp = fixedIp;
-            this.hostIp = hostIp;
-        }
-    }
 }
\ No newline at end of file
--- a/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingRulePopulator.java
View file @5df6518
+++ b/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingRulePopulator.java
View file @5df6518
@@ -18,7 +18,6 @@ package org.onosproject.openstackswitching;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ip4Address;
-import org.onlab.packet.MacAddress;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.net.Device;
 import org.onosproject.net.DeviceId;
@@ -45,6 +44,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import java.util.Collection;
+import java.util.Map;
 /**
  * Populates switching flow rules.
@@ -118,31 +118,25 @@ public class OpenstackSwitchingRulePopulator {
         }
     }
-    /**
+    private void setFlowRuleForTunnelTag(DeviceId deviceId, Port port, String vni) {
-     * Returns OpenstackPort object for the Port reference given.
-     *
-     * @param port Port object
-     * @return OpenstackPort reference, or null when not found
-     */
-    public OpenstackPort openstackPort(Port port) {
-        String uuid = port.annotations().value("portName").substring(3);
-        return openstackPortList.stream().filter(p -> p.id().startsWith(uuid))
-                .findAny().orElse(null);
-    }
-    /**
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-     * Remove flows rules for the VM removed.
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-     *
+
-     * @param deviceId device to remove rules
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-     * @param vmIp IP address of the VM removed
+                .matchInPort(port.number());
-     */
+
-    public void removeSwitchingRules(DeviceId deviceId, Ip4Address vmIp) {
+        tBuilder.setTunnelId(Long.parseLong(vni));
-        removeFlowRuleForVMsInSameCnode(deviceId, vmIp);
+
-        deviceService.getAvailableDevices().forEach(device -> {
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
-            if (!device.id().equals(deviceId)) {
+                .withSelector(sBuilder.build())
-                removeVxLanFlowRule(device.id(), vmIp);
+                .withTreatment(tBuilder.build())
-            }
+                .withPriority(TUNNELTAG_RULE_PRIORITY)
-        });
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(deviceId, fo);
     }
     /**
@@ -155,14 +149,45 @@ public class OpenstackSwitchingRulePopulator {
         Ip4Address vmIp = getFixedIpAddressForPort(port.annotations().value("portName"));
         String portName = port.annotations().value("portName");
         String vni = getVniForPort(portName);
-        MacAddress vmMacAddress = getVmMacAddressForPort(portName);
         if (vmIp != null) {
-            setFlowRuleForVMsInSameCnode(vmIp, device.id(), port, vni, vmMacAddress);
+            setFlowRuleForVMsInSameCnode(vmIp, device.id(), port, vni);
         }
     }
     /**
+     * Sets the flow rules for traffic between VMs in the same Cnode.
+     *
+     * @param ip4Address VM IP address
+     * @param id device ID to put rules
+     * @param port VM port
+     * @param vni VM VNI
+     */
+    private void setFlowRuleForVMsInSameCnode(Ip4Address ip4Address, DeviceId id,
+                                              Port port, String vni) {
+
+        //For L2 Switching Case
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(ip4Address.toIpPrefix())
+                .matchTunnelId(Long.parseLong(vni));
+
+        tBuilder.setOutput(port.number());
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(SWITCHING_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(id, fo);
+    }
+
+    /**
      * Populates the flow rules for traffic to VMs in different Cnode using
      * Nicira extention.
      *
@@ -174,7 +199,6 @@ public class OpenstackSwitchingRulePopulator {
         String channelId = device.annotations().value("channelId");
         Ip4Address hostIpAddress = Ip4Address.valueOf(channelId.split(":")[0]);
         Ip4Address fixedIp = getFixedIpAddressForPort(portName);
-        MacAddress vmMac = getVmMacAddressForPort(portName);
         String vni = getVniForPort(portName);
         deviceService.getAvailableDevices().forEach(d -> {
             if (!d.equals(device)) {
@@ -183,11 +207,10 @@ public class OpenstackSwitchingRulePopulator {
                     if (!p.equals(port) && vni.equals(getVniForPort(pName))) {
                         String cidx = d.annotations().value("channelId");
                         Ip4Address hostIpx = Ip4Address.valueOf(cidx.split(":")[0]);
-                        MacAddress vmMacx = getVmMacAddressForPort(pName);
                         Ip4Address fixedIpx = getFixedIpAddressForPort(pName);
                         if (port.isEnabled()) {
-                            setVxLanFlowRule(vni, device.id(), hostIpx, fixedIpx, vmMacx);
+                            setVxLanFlowRule(vni, device.id(), hostIpx, fixedIpx);
-                            setVxLanFlowRule(vni, d.id(), hostIpAddress, fixedIp, vmMac);
+                            setVxLanFlowRule(vni, d.id(), hostIpAddress, fixedIp);
                         }
                     }
                 });
@@ -196,88 +219,80 @@ public class OpenstackSwitchingRulePopulator {
     }
     /**
-     * Returns the VNI of the VM of the port.
+     * Sets the flow rules between traffic from VMs in different Cnode.
      *
-     * @param portName VM port
+     * @param vni  VNI
-     * @return VNI
+     * @param deviceId device ID
+     * @param hostIp host IP of the VM
+     * @param vmIp fixed IP of the VM
      */
-    private String getVniForPort(String portName) {
+    private void setVxLanFlowRule(String vni, DeviceId deviceId, Ip4Address hostIp,
-        String uuid = portName.substring(3);
+                                  Ip4Address vmIp) {
-        OpenstackPort port = openstackPortList.stream()
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-                .filter(p -> p.id().startsWith(uuid))
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-                .findAny().orElse(null);
-        if (port == null) {
-            log.debug("No port information for port {}", portName);
-            return null;
-        }
-        OpenstackNetwork network = openstackNetworkList.stream()
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .filter(n -> n.id().equals(port.networkId()))
+                .matchTunnelId(Long.parseLong(vni))
-                .findAny().orElse(null);
+                .matchIPDst(vmIp.toIpPrefix());
-        if (network == null) {
+        tBuilder.extension(buildNiciraExtenstion(deviceId, hostIp), deviceId)
-            log.warn("No VNI information for network {}", port.networkId());
+                .setOutput(getTunnelPort(deviceId));
-            return null;
-        }
-        return network.segmentId();
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(SWITCHING_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(deviceId, fo);
     }
     /**
-     * Returns the Fixed IP address of the VM.
+     * Returns OpenstackPort object for the Port reference given.
      *
-     * @param portName VM port info
+     * @param port Port object
-     * @return IP address of the VM
+     * @return OpenstackPort reference, or null when not found
      */
-    private Ip4Address getFixedIpAddressForPort(String portName) {
+    public OpenstackPort openstackPort(Port port) {
-
+        String uuid = port.annotations().value("portName").substring(3);
-        String uuid = portName.substring(3);
+        return openstackPortList.stream().filter(p -> p.id().startsWith(uuid))
-        OpenstackPort port = openstackPortList.stream()
+                .findAny().orElse(null);
-                .filter(p -> p.id().startsWith(uuid))
-                .findFirst().orElse(null);
-
-        if (port == null) {
-            log.error("There is no port information for port name {}", portName);
-            return null;
-        }
-
-        if (port.fixedIps().isEmpty()) {
-            log.error("There is no fixed IP info in the port information");
-            return null;
-        }
-
-        return (Ip4Address) port.fixedIps().values().toArray()[0];
     }
     /**
-     * Returns the MAC address of the VM of the port.
+     * Remove flows rules for the removed VM.
      *
-     * @param portName VM port
+     * @param removedPort removedport info
-     * @return MAC address of the VM
+     * @param openstackPortInfoMap openstackPortInfoMap
      */
-    private MacAddress getVmMacAddressForPort(String portName) {
+    public void removeSwitchingRules(Port removedPort, Map<String, OpenstackPortInfo> openstackPortInfoMap) {
-        String uuid = portName.substring(3);
+        OpenstackPortInfo openstackPortInfo = openstackPortInfoMap
-        OpenstackPort port = openstackPortList.stream()
+                .get(removedPort.annotations().value("portName"));
-                .filter(p -> p.id().startsWith(uuid))
-                .findFirst().orElse(null);
-        if (port == null) {
+        DeviceId deviceId = openstackPortInfo.deviceId();
-            log.error("There is no port information for port name {}", portName);
+        Ip4Address vmIp = openstackPortInfo.ip();
-            return null;
+        PortNumber portNumber = removedPort.number();
-        }
+        long vni = openstackPortInfo.vni();
-        return port.macAddress();
+        removeFlowRuleForTunnelTag(deviceId, portNumber);
+        removeFlowRuleForVMsInSameCnode(deviceId, vmIp, vni);
+        removeFlowRuleForVMsInDiffrentCnode(removedPort, deviceId, vmIp, vni, openstackPortInfoMap);
     }
-    private void setFlowRuleForTunnelTag(DeviceId deviceId, Port port, String vni) {
+    /**
-
+     * Removes flow rules for tagging tunnelId.
+     *
+     * @param deviceId device id
+     * @param portNumber port number
+     */
+    private void removeFlowRuleForTunnelTag(DeviceId deviceId, PortNumber portNumber) {
         TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
         TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
         sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchInPort(port.number());
+                .matchInPort(portNumber);
-
-        tBuilder.setTunnelId(Long.parseLong(vni));
         ForwardingObjective fo = DefaultForwardingObjective.builder()
                 .withSelector(sBuilder.build())
@@ -285,110 +300,159 @@ public class OpenstackSwitchingRulePopulator {
                 .withPriority(TUNNELTAG_RULE_PRIORITY)
                 .withFlag(ForwardingObjective.Flag.SPECIFIC)
                 .fromApp(appId)
-                .add();
+                .remove();
         flowObjectiveService.forward(deviceId, fo);
     }
-
     /**
-     * Sets the flow rules for traffic between VMs in the same Cnode.
+     * Removes the flow rules for traffic between VMs in the same Cnode.
      *
-     * @param ip4Address VM IP address
+     * @param deviceId device id on which removed VM was run
-     * @param id device ID to put rules
+     * @param vmIp ip of the removed VM
-     * @param port VM port
+     * @param vni vni which removed VM was belonged
-     * @param vni VM VNI
-     * @param vmMacAddress VM MAC address
      */
-    private void setFlowRuleForVMsInSameCnode(Ip4Address ip4Address, DeviceId id,
+    private void removeFlowRuleForVMsInSameCnode(DeviceId deviceId, Ip4Address vmIp, long vni) {
-                                              Port port, String vni, MacAddress vmMacAddress) {
-
-        //For L2 Switching Case
         TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
         sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchIPDst(ip4Address.toIpPrefix())
+                .matchIPDst(vmIp.toIpPrefix())
-                .matchTunnelId(Long.parseLong(vni));
+                .matchTunnelId(vni);
-
-        tBuilder.setOutput(port.number());
         ForwardingObjective fo = DefaultForwardingObjective.builder()
                 .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
+                .withTreatment(DefaultTrafficTreatment.builder().build())
-                .withPriority(SWITCHING_RULE_PRIORITY)
                 .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .withPriority(SWITCHING_RULE_PRIORITY)
                 .fromApp(appId)
-                .add();
+                .remove();
-        flowObjectiveService.forward(id, fo);
+        flowObjectiveService.forward(deviceId, fo);
     }
-
     /**
-     * Sets the flow rules between traffic from VMs in different Cnode.
+     * Removes the flow rules for traffic between VMs in the different Cnode.
      *
-     * @param vni  VNI
+     * @param removedPort removedport info
-     * @param id device ID
+     * @param deviceId device id on which removed VM was run
-     * @param hostIp host IP of the VM
+     * @param vmIp ip of the removed VM
-     * @param vmIp fixed IP of the VM
+     * @param vni vni which removed VM was belonged
-     * @param vmMac MAC address of the VM
+     * @param openstackPortInfoMap openstackPortInfoMap
      */
-    private void setVxLanFlowRule(String vni, DeviceId id, Ip4Address hostIp,
+    private void removeFlowRuleForVMsInDiffrentCnode(Port removedPort, DeviceId deviceId, Ip4Address vmIp,
-                                  Ip4Address vmIp, MacAddress vmMac) {
+                                                     long vni, Map<String, OpenstackPortInfo> openstackPortInfoMap) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchTunnelId(Long.parseLong(vni))
-                .matchIPDst(vmIp.toIpPrefix());
-        tBuilder.extension(buildNiciraExtenstion(id, hostIp), id)
+        final boolean anyPortRemainedInSameCnode
-                .setOutput(getTunnelPort(id));
+                = checkIfAnyPortRemainedInSameCnode(removedPort, deviceId, vni, openstackPortInfoMap);
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
+        openstackPortInfoMap.forEach((port, portInfo) -> {
-                .withSelector(sBuilder.build())
+            if (portInfo.vni() == vni && !portInfo.deviceId().equals(deviceId)) {
-                .withTreatment(tBuilder.build())
+                removeVxLanFlowRule(portInfo.deviceId(), vmIp, vni);
-                .withPriority(SWITCHING_RULE_PRIORITY)
+                if (!anyPortRemainedInSameCnode) {
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                    removeVxLanFlowRule(deviceId, portInfo.ip(), vni);
-                .fromApp(appId)
+                }
-                .add();
+            }
-
+        });
-        flowObjectiveService.forward(id, fo);
     }
-    private void removeFlowRuleForVMsInSameCnode(DeviceId id, Ip4Address vmIp) {
+    /**
+     * Removes the flow rules between traffic from VMs in different Cnode.
+     *
+     * @param deviceId device id
+     * @param vmIp ip
+     * @param vni vni which removed VM was belonged
+     */
+    private void removeVxLanFlowRule(DeviceId deviceId, Ip4Address vmIp, long vni) {
         TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
         sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(vni)
                 .matchIPDst(vmIp.toIpPrefix());
         ForwardingObjective fo = DefaultForwardingObjective.builder()
                 .withSelector(sBuilder.build())
                 .withTreatment(DefaultTrafficTreatment.builder().build())
-                .withFlag(ForwardingObjective.Flag.VERSATILE)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
                 .withPriority(SWITCHING_RULE_PRIORITY)
                 .fromApp(appId)
                 .remove();
-        flowObjectiveService.forward(id, fo);
+        flowObjectiveService.forward(deviceId, fo);
     }
-    private void removeVxLanFlowRule(DeviceId id, Ip4Address vmIp) {
+    /**
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+     * Checks if there is any port remained with same vni at the device, on which the removed VM was run.
-        // XXX: Later, more matches will be added when multiple table is implemented.
+     *
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+     * @param removedPort removedport info
-                .matchIPDst(vmIp.toIpPrefix());
+     * @param deviceId device id on which removed VM was run
+     * @param vni vni which removed VM was belonged
+     * @param openstackPortInfoMap openstackPortInfoMap
+     * @return true if there is, false otherwise
+     */
+    private boolean checkIfAnyPortRemainedInSameCnode(Port removedPort, DeviceId deviceId, long vni,
+                                                    Map<String, OpenstackPortInfo> openstackPortInfoMap) {
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
+        for (Map.Entry<String, OpenstackPortInfo> entry : openstackPortInfoMap.entrySet()) {
-                .withSelector(sBuilder.build())
+            if (!removedPort.annotations().value("portName").equals(entry.getKey())) {
-                .withTreatment(DefaultTrafficTreatment.builder().build())
+                if (entry.getValue().vni() == vni && entry.getValue().deviceId().equals(deviceId)) {
-                .withFlag(ForwardingObjective.Flag.VERSATILE)
+                    return true;
-                .withPriority(SWITCHING_RULE_PRIORITY)
+                }
-                .fromApp(appId)
+            }
-                .remove();
+        }
+        return false;
+    }
-        flowObjectiveService.forward(id, fo);
+    /**
+     * Returns the VNI of the VM of the port.
+     *
+     * @param portName VM port
+     * @return VNI
+     */
+    private String getVniForPort(String portName) {
+        String uuid = portName.substring(3);
+        OpenstackPort port = openstackPortList.stream()
+                .filter(p -> p.id().startsWith(uuid))
+                .findAny().orElse(null);
+        if (port == null) {
+            log.debug("No port information for port {}", portName);
+            return null;
+        }
+
+        OpenstackNetwork network = openstackNetworkList.stream()
+                .filter(n -> n.id().equals(port.networkId()))
+                .findAny().orElse(null);
+        if (network == null) {
+            log.warn("No VNI information for network {}", port.networkId());
+            return null;
+        }
+
+        return network.segmentId();
+    }
+
+    /**
+     * Returns the Fixed IP address of the VM.
+     *
+     * @param portName VM port info
+     * @return IP address of the VM
+     */
+    private Ip4Address getFixedIpAddressForPort(String portName) {
+
+        String uuid = portName.substring(3);
+        OpenstackPort port = openstackPortList.stream()
+                .filter(p -> p.id().startsWith(uuid))
+                .findFirst().orElse(null);
+
+        if (port == null) {
+            log.error("There is no port information for port name {}", portName);
+            return null;
+        }
+
+        if (port.fixedIps().isEmpty()) {
+            log.error("There is no fixed IP info in the port information");
+            return null;
+        }
+
+        return (Ip4Address) port.fixedIps().values().toArray()[0];
     }
     private ExtensionTreatment buildNiciraExtenstion(DeviceId id, Ip4Address hostIp) {
@@ -409,8 +473,8 @@ public class OpenstackSwitchingRulePopulator {
         return  extensionInstruction;
     }
-    private PortNumber getTunnelPort(DeviceId id) {
+    private PortNumber getTunnelPort(DeviceId deviceId) {
-        Port port = deviceService.getPorts(id).stream()
+        Port port = deviceService.getPorts(deviceId).stream()
                 .filter(p -> p.annotations().value("portName").equals("vxlan"))
                 .findAny().orElse(null);