[ONOS-4428] Fix the L3 dataflow bug when in same ternant but different

subnet with different routers, in the odler version of ONOS, in the same ternant, different network with differnent routers, dataflows can be send to each other successfully. Change-Id: I6e50289023711eb0f6005efee0efd6f7ab9ec3ee

[ONOS-4428] Fix the L3 dataflow bug when in same ternant but different
subnet with different routers, in the odler version of ONOS, in the same ternant, different network with differnent routers, dataflows can be send to each other successfully. Change-Id: I6e50289023711eb0f6005efee0efd6f7ab9ec3ee
lishuai
Commit 8798bbe63339ad4985fea8b3f9687511f6facc04 8798bbe6 1 parent e85d1b98
Showing 5 changed files with 187 additions and 42 deletions
apps/vtn/sfcmgr/src/test/java/org/onosproject/sfc/util/VtnRscAdapter.java
apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/manager/impl/VtnManager.java
apps/vtn/vtnrsc/src/main/java/org/onosproject/vtnrsc/TenantRouter.java
apps/vtn/vtnrsc/src/main/java/org/onosproject/vtnrsc/service/VtnRscService.java
apps/vtn/vtnrsc/src/main/java/org/onosproject/vtnrsc/service/impl/VtnRscManager.java
--- a/apps/vtn/sfcmgr/src/test/java/org/onosproject/sfc/util/VtnRscAdapter.java
View file @8798bbe
+++ b/apps/vtn/sfcmgr/src/test/java/org/onosproject/sfc/util/VtnRscAdapter.java
View file @8798bbe
@@ -24,6 +24,7 @@ import org.onosproject.net.Host;
 import org.onosproject.net.HostId;
 import org.onosproject.vtnrsc.SegmentationId;
 import org.onosproject.vtnrsc.TenantId;
+import org.onosproject.vtnrsc.TenantRouter;
 import org.onosproject.vtnrsc.VirtualPortId;
 import org.onosproject.vtnrsc.event.VtnRscListener;
 import org.onosproject.vtnrsc.service.VtnRscService;
@@ -79,4 +80,9 @@ public class VtnRscAdapter implements VtnRscService {
     public void removeDeviceIdOfOvsMap(Host host, TenantId tenantId,
                                        DeviceId deviceId) {
     }
+
+    @Override
+    public SegmentationId getL3vni(TenantRouter tenantRouter) {
+        return null;
+    }
 }
--- a/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/manager/impl/VtnManager.java
View file @8798bbe
+++ b/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/manager/impl/VtnManager.java
View file @8798bbe
@@ -110,6 +110,7 @@ import org.onosproject.vtnrsc.SubnetId;
 import org.onosproject.vtnrsc.TenantId;
 import org.onosproject.vtnrsc.TenantNetwork;
 import org.onosproject.vtnrsc.TenantNetworkId;
+import org.onosproject.vtnrsc.TenantRouter;
 import org.onosproject.vtnrsc.VirtualPort;
 import org.onosproject.vtnrsc.VirtualPortId;
 import org.onosproject.vtnrsc.event.VtnRscEvent;
@@ -199,7 +200,7 @@ public class VtnManager implements VtnService {
     private static final String VIRTUALPORT = "vtn-virtual-port";
     private static final String SWITCHES_OF_CONTROLLER = "switchesOfController";
     private static final String SWITCH_OF_LOCAL_HOST_PORTS = "switchOfLocalHostPorts";
-    private static final String ROUTERINF_FLAG_OF_TENANT = "routerInfFlagOfTenant";
+    private static final String ROUTERINF_FLAG_OF_TENANTROUTER = "routerInfFlagOfTenantRouter";
     private static final String HOSTS_OF_SUBNET = "hostsOfSubnet";
     private static final String EX_PORT_OF_DEVICE = "exPortOfDevice";
     private static final String EX_PORT_MAP = "exPortMap";
@@ -210,7 +211,7 @@ public class VtnManager implements VtnService {
     private EventuallyConsistentMap<IpAddress, Boolean> switchesOfController;
     private EventuallyConsistentMap<DeviceId, NetworkOfLocalHostPorts> switchOfLocalHostPorts;
     private EventuallyConsistentMap<SubnetId, Map<HostId, Host>> hostsOfSubnet;
-    private EventuallyConsistentMap<TenantId, Boolean> routerInfFlagOfTenant;
+    private EventuallyConsistentMap<TenantRouter, Boolean> routerInfFlagOfTenantRouter;
     private EventuallyConsistentMap<DeviceId, Port> exPortOfDevice;
     private static ConsistentMap<String, String> exPortMap;
@@ -269,9 +270,9 @@ public class VtnManager implements VtnService {
                 .withTimestampProvider((k, v) -> clockService.getTimestamp())
                 .build();
-        routerInfFlagOfTenant = storageService
+        routerInfFlagOfTenantRouter = storageService
-                .<TenantId, Boolean>eventuallyConsistentMapBuilder()
+                .<TenantRouter, Boolean>eventuallyConsistentMapBuilder()
-                .withName(ROUTERINF_FLAG_OF_TENANT).withSerializer(serializer)
+                .withName(ROUTERINF_FLAG_OF_TENANTROUTER).withSerializer(serializer)
                 .withTimestampProvider((k, v) -> clockService.getTimestamp())
                 .build();
@@ -766,10 +767,13 @@ public class VtnManager implements VtnService {
         vPortStore.put(gwPort.portId(), gwPort);
         Iterable<RouterInterface> interfaces = routerInterfaceService
                 .getRouterInterfaces();
-        Set<RouterInterface> interfacesSet = Sets.newHashSet(interfaces)
+        Set<RouterInterface> interfacesSet = Sets.newHashSet(interfaces).stream()
-                .stream().filter(r -> r.tenantId().equals(routerInf.tenantId()))
+                .filter(r -> r.tenantId().equals(routerInf.tenantId()))
+                .filter(r -> r.routerId().equals(routerInf.routerId()))
                 .collect(Collectors.toSet());
-        if (routerInfFlagOfTenant.get(routerInf.tenantId()) != null) {
+        TenantRouter tenantRouter = TenantRouter
+                .tenantRouter(routerInf.tenantId(), routerInf.routerId());
+        if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
             programRouterInterface(routerInf, operation);
         } else {
             if (interfacesSet.size() >= SUBNET_NUM) {
@@ -787,10 +791,12 @@ public class VtnManager implements VtnService {
         Set<RouterInterface> interfacesSet = Sets.newHashSet(interfaces)
                 .stream().filter(r -> r.tenantId().equals(routerInf.tenantId()))
                 .collect(Collectors.toSet());
-        if (routerInfFlagOfTenant.get(routerInf.tenantId()) != null) {
+        TenantRouter tenantRouter = TenantRouter
+                .tenantRouter(routerInf.tenantId(), routerInf.routerId());
+        if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
             programRouterInterface(routerInf, operation);
             if (interfacesSet.size() == 1) {
-                routerInfFlagOfTenant.remove(routerInf.tenantId());
+                routerInfFlagOfTenantRouter.remove(tenantRouter);
                 interfacesSet.stream().forEach(r -> {
                     programRouterInterface(r, operation);
                 });
@@ -822,7 +828,9 @@ public class VtnManager implements VtnService {
             if (hosts != null && hosts.size() > 0) {
                 subnetVmNum++;
                 if (subnetVmNum >= SUBNET_NUM) {
-                    routerInfFlagOfTenant.put(r.tenantId(), true);
+                    TenantRouter tenantRouter = TenantRouter
+                            .tenantRouter(r.tenantId(), r.routerId());
+                    routerInfFlagOfTenantRouter.put(tenantRouter, true);
                     interfacesSet.stream().forEach(f -> {
                         programRouterInterface(f, operation);
                     });
@@ -834,7 +842,9 @@ public class VtnManager implements VtnService {
     private void programRouterInterface(RouterInterface routerInf,
                                         Objective.Operation operation) {
-        SegmentationId l3vni = vtnRscService.getL3vni(routerInf.tenantId());
+        TenantRouter tenantRouter = TenantRouter
+                .tenantRouter(routerInf.tenantId(), routerInf.routerId());
+        SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
         // Get all the host of the subnet
         Map<HostId, Host> hosts = hostsOfSubnet.get(routerInf.subnetId());
         hosts.values().stream().forEach(h -> {
@@ -929,8 +939,9 @@ public class VtnManager implements VtnService {
             if (host != null && vmPort != null && fipPort != null) {
                 DeviceId deviceId = host.location().deviceId();
                 Port exPort = exPortOfDevice.get(deviceId);
-                SegmentationId l3vni = vtnRscService
+                TenantRouter tenantRouter = TenantRouter
-                        .getL3vni(vmPort.tenantId());
+                        .tenantRouter(floaingIp.tenantId(), floaingIp.routerId());
+                SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
                 // Floating ip BIND
                 if (type == VtnRscEvent.Type.FLOATINGIP_BIND) {
                     vPortStore.put(fipPort.portId(), fipPort);
@@ -951,7 +962,7 @@ public class VtnManager implements VtnService {
     private void applyNorthSouthL3Flows(DeviceId deviceId, Host host,
                                         VirtualPort vmPort, VirtualPort fipPort,
                                         FloatingIp floatingIp,
-                                        SegmentationId l3Vni, Port exPort,
+                                        SegmentationId l3vni, Port exPort,
                                         Objective.Operation operation) {
         if (!mastershipService.isLocalMaster(deviceId)) {
             log.debug("not master device:{}", deviceId);
@@ -980,9 +991,9 @@ public class VtnManager implements VtnService {
                                          operation);
         dnatService.programRules(deviceId, floatingIp.floatingIp(),
                                      fGwMac, floatingIp.fixedIp(),
-                                     l3Vni, operation);
+                                     l3vni, operation);
         l3ForwardService
-                .programRouteRules(deviceId, l3Vni, floatingIp.fixedIp(),
+                .programRouteRules(deviceId, l3vni, floatingIp.fixedIp(),
                                    vmNetwork.segmentationId(), dstVmGwMac,
                                    vmPort.macAddress(), operation);
@@ -990,8 +1001,8 @@ public class VtnManager implements VtnService {
         classifierService.programL3InPortClassifierRules(deviceId,
                                                          host.location().port(),
                                                          host.mac(), dstVmGwMac,
-                                                         l3Vni, operation);
+                                                         l3vni, operation);
-        snatService.programRules(deviceId, l3Vni, floatingIp.fixedIp(),
+        snatService.programRules(deviceId, l3vni, floatingIp.fixedIp(),
                                      fGwMac, exPortMac,
                                      floatingIp.floatingIp(),
                                      fipNetwork.segmentationId(), operation);
@@ -1056,7 +1067,6 @@ public class VtnManager implements VtnService {
         }
         TenantId tenantId = port.tenantId();
         Port exPort = exPortOfDevice.get(deviceId);
-        SegmentationId l3vni = vtnRscService.getL3vni(tenantId);
         Iterator<FixedIp> fixips = port.fixedIps().iterator();
         SubnetId sid = null;
         IpAddress hostIp = null;
@@ -1069,26 +1079,36 @@ public class VtnManager implements VtnService {
         // L3 internal network access to each other
         Iterable<RouterInterface> interfaces = routerInterfaceService
                 .getRouterInterfaces();
-        Set<RouterInterface> interfacesSet = Sets.newHashSet(interfaces)
+        Set<RouterInterface> hostInterfaces = Sets.newHashSet(interfaces)
                 .stream().filter(r -> r.tenantId().equals(tenantId))
+                .filter(r -> r.subnetId().equals(subnetId))
                 .collect(Collectors.toSet());
-        long count = interfacesSet.stream()
+        hostInterfaces.stream().forEach(routerInf -> {
-                .filter(r -> !r.subnetId().equals(subnetId)).count();
+            Set<RouterInterface> interfacesSet = Sets.newHashSet(interfaces)
-        if (count > 0) {
+                    .stream().filter(r -> r.tenantId().equals(tenantId))
-            if (operation == Objective.Operation.ADD) {
+                    .filter(r -> r.routerId().equals(routerInf.routerId()))
-                if (routerInfFlagOfTenant.get(tenantId) != null) {
+                    .collect(Collectors.toSet());
-                    applyEastWestL3Flows(host, l3vni, operation);
+            long count = interfacesSet.stream()
-                } else {
+                    .filter(r -> !r.subnetId().equals(subnetId)).count();
-                    if (interfacesSet.size() > 1) {
+            if (count > 0) {
-                        programInterfacesSet(interfacesSet, operation);
+                TenantRouter tenantRouter = TenantRouter
+                        .tenantRouter(routerInf.tenantId(), routerInf.routerId());
+                SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
+                if (operation == Objective.Operation.ADD) {
+                    if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
+                        applyEastWestL3Flows(host, l3vni, operation);
+                    } else {
+                        if (interfacesSet.size() > 1) {
+                            programInterfacesSet(interfacesSet, operation);
+                        }
+                    }
+                } else if (operation == Objective.Operation.REMOVE) {
+                    if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
+                        applyEastWestL3Flows(host, l3vni, operation);
                     }
-                }
-            } else if (operation == Objective.Operation.REMOVE) {
-                if (routerInfFlagOfTenant.get(tenantId) != null) {
-                    applyEastWestL3Flows(host, l3vni, operation);
                 }
             }
-        }
+        });
         // L3 external and internal network access to each other
         FloatingIp floatingIp = null;
         Iterable<FloatingIp> floatingIps = floatingIpService.getFloatingIps();
@@ -1103,6 +1123,9 @@ public class VtnManager implements VtnService {
             }
         }
         if (floatingIp != null) {
+            TenantRouter tenantRouter = TenantRouter
+                    .tenantRouter(floatingIp.tenantId(), floatingIp.routerId());
+            SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
             VirtualPort fipPort = virtualPortService
                     .getPort(floatingIp.networkId(), floatingIp.floatingIp());
             if (fipPort == null) {
--- a/apps/vtn/vtnrsc/src/main/java/org/onosproject/vtnrsc/TenantRouter.java 0 → 100644
View file @8798bbe
+++ b/apps/vtn/vtnrsc/src/main/java/org/onosproject/vtnrsc/TenantRouter.java 0 → 100644
View file @8798bbe
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.vtnrsc;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Objects;
+
+public final class TenantRouter {
+    private final TenantId tenantId;
+    private final RouterId routerId;
+
+    /**
+     * Construct a TenantRouter object.
+     *
+     * @param tenantId  the tenant identifier
+     * @param routerId  router identifier
+     */
+    private TenantRouter(TenantId tenantId, RouterId routerId) {
+        this.tenantId = checkNotNull(tenantId, "tenantId cannot be null");
+        this.routerId = checkNotNull(routerId, "routerId cannot be null");
+    }
+
+    /**
+     * Create a TenantRouter object.
+     *
+     * @param tenantId  the tenant identifier
+     * @param routerId  router identifier
+     * @return TenantRouter
+     */
+    public static TenantRouter tenantRouter(TenantId tenantId, RouterId routerId) {
+        return new TenantRouter(tenantId, routerId);
+    }
+
+    public TenantId tenantId() {
+        return tenantId;
+    }
+
+    public RouterId routerId() {
+        return routerId;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(tenantId, routerId);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj instanceof TenantRouter) {
+            final TenantRouter that = (TenantRouter) obj;
+            return this.getClass() == that.getClass()
+                    && Objects.equals(this.tenantId, that.tenantId)
+                    && Objects.equals(this.routerId, that.routerId);
+        }
+        return false;
+    }
+
+    @Override
+    public String toString() {
+        return toStringHelper(this)
+                .add("tenantId", tenantId)
+                .add("routerId", routerId)
+                .toString();
+    }
+}
--- a/apps/vtn/vtnrsc/src/main/java/org/onosproject/vtnrsc/service/VtnRscService.java
View file @8798bbe
+++ b/apps/vtn/vtnrsc/src/main/java/org/onosproject/vtnrsc/service/VtnRscService.java
View file @8798bbe
@@ -23,6 +23,7 @@ import org.onosproject.net.Host;
 import org.onosproject.net.HostId;
 import org.onosproject.vtnrsc.SegmentationId;
 import org.onosproject.vtnrsc.TenantId;
+import org.onosproject.vtnrsc.TenantRouter;
 import org.onosproject.vtnrsc.VirtualPortId;
 import org.onosproject.vtnrsc.event.VtnRscEvent;
 import org.onosproject.vtnrsc.event.VtnRscListener;
@@ -42,6 +43,14 @@ public interface VtnRscService extends ListenerService<VtnRscEvent, VtnRscListen
     SegmentationId getL3vni(TenantId tenantId);
     /**
+     * Returns the SegmentationId of tenantRouter.
+     *
+     * @param tenantRouter TenantRouter
+     * @return SegmentationId the SegmentationId of tenantRouter
+     */
+    SegmentationId getL3vni(TenantRouter tenantRouter);
+
+    /**
      * Returns Classifier Ovs list of the specific tenant.
      *
      * @param tenantId tenant identifier
--- a/apps/vtn/vtnrsc/src/main/java/org/onosproject/vtnrsc/service/impl/VtnRscManager.java
View file @8798bbe
+++ b/apps/vtn/vtnrsc/src/main/java/org/onosproject/vtnrsc/service/impl/VtnRscManager.java
View file @8798bbe
@@ -56,6 +56,7 @@ import org.onosproject.vtnrsc.SegmentationId;
 import org.onosproject.vtnrsc.Subnet;
 import org.onosproject.vtnrsc.SubnetId;
 import org.onosproject.vtnrsc.TenantId;
+import org.onosproject.vtnrsc.TenantRouter;
 import org.onosproject.vtnrsc.VirtualPort;
 import org.onosproject.vtnrsc.VirtualPortId;
 import org.onosproject.vtnrsc.event.VtnRscEvent;
@@ -111,7 +112,8 @@ public class VtnRscManager extends AbstractListenerManager<VtnRscEvent, VtnRscLi
     private FlowClassifierListener flowClassifierListener = new InnerFlowClassifierListener();
     private PortChainListener portChainListener = new InnerPortChainListener();
-    private EventuallyConsistentMap<TenantId, SegmentationId> l3vniMap;
+    private EventuallyConsistentMap<TenantId, SegmentationId> l3vniTenantMap;
+    private EventuallyConsistentMap<TenantRouter, SegmentationId> l3vniTenantRouterMap;
     private EventuallyConsistentMap<TenantId, Set<DeviceId>> classifierOvsMap;
     private EventuallyConsistentMap<TenantId, Set<DeviceId>> sffOvsMap;
@@ -122,7 +124,8 @@ public class VtnRscManager extends AbstractListenerManager<VtnRscEvent, VtnRscLi
     private static final String DEVICEID_NOT_NULL = "deviceId cannot be null";
     private static final String VIRTUALPORTID_NOT_NULL = "virtualPortId cannot be null";
     private static final String HOST_NOT_NULL = "host cannot be null";
-    private static final String L3VNIMAP = "l3vniMap";
+    private static final String L3VNITENANTMAP = "l3vniTenantMap";
+    private static final String L3VNITENANTROUTERMAP = "l3vniTenantRouterMap";
     private static final String CLASSIFIEROVSMAP = "classifierOvsMap";
     private static final String SFFOVSMAP = "sffOvsMap";
@@ -165,9 +168,15 @@ public class VtnRscManager extends AbstractListenerManager<VtnRscEvent, VtnRscLi
         KryoNamespace.Builder serializer = KryoNamespace.newBuilder()
                 .register(KryoNamespaces.API)
                 .register(TenantId.class, DeviceId.class, SegmentationId.class);
-        l3vniMap = storageService
+        l3vniTenantMap = storageService
                 .<TenantId, SegmentationId>eventuallyConsistentMapBuilder()
-                .withName(L3VNIMAP).withSerializer(serializer)
+                .withName(L3VNITENANTMAP).withSerializer(serializer)
+                .withTimestampProvider((k, v) -> clockService.getTimestamp())
+                .build();
+
+        l3vniTenantRouterMap = storageService
+                .<TenantRouter, SegmentationId>eventuallyConsistentMapBuilder()
+                .withName(L3VNITENANTROUTERMAP).withSerializer(serializer)
                 .withTimestampProvider((k, v) -> clockService.getTimestamp())
                 .build();
@@ -195,7 +204,8 @@ public class VtnRscManager extends AbstractListenerManager<VtnRscEvent, VtnRscLi
         flowClassifierService.removeListener(flowClassifierListener);
         portChainService.removeListener(portChainListener);
-        l3vniMap.destroy();
+        l3vniTenantMap.destroy();
+        l3vniTenantRouterMap.destroy();
         classifierOvsMap.destroy();
         sffOvsMap.destroy();
         log.info("Stopped");
@@ -204,13 +214,27 @@ public class VtnRscManager extends AbstractListenerManager<VtnRscEvent, VtnRscLi
     @Override
     public SegmentationId getL3vni(TenantId tenantId) {
         checkNotNull(tenantId, "tenantId cannot be null");
-        SegmentationId l3vni = l3vniMap.get(tenantId);
+        SegmentationId l3vni = l3vniTenantMap.get(tenantId);
+        if (l3vni == null) {
+            long segmentationId = coreService.getIdGenerator(RUNNELOPTOPOIC)
+                    .getNewId();
+            l3vni = SegmentationId.segmentationId(String
+                    .valueOf(segmentationId));
+            l3vniTenantMap.put(tenantId, l3vni);
+        }
+        return l3vni;
+    }
+
+    @Override
+    public SegmentationId getL3vni(TenantRouter tenantRouter) {
+        checkNotNull(tenantRouter, "tenantRouter cannot be null");
+        SegmentationId l3vni = l3vniTenantRouterMap.get(tenantRouter);
         if (l3vni == null) {
             long segmentationId = coreService.getIdGenerator(RUNNELOPTOPOIC)
                     .getNewId();
             l3vni = SegmentationId.segmentationId(String
                     .valueOf(segmentationId));
-            l3vniMap.put(tenantId, l3vni);
+            l3vniTenantRouterMap.put(tenantRouter, l3vni);
         }
         return l3vni;
     }