[ONOS-4481] Integrate scalable gateway to openstackRouting

- Modified openstack_routing to use scalable gateway application - Add GatewayNode Builder method Change-Id: Ib12d52009eeb89ab59f01cafe794894aac80985e

[ONOS-4481] Integrate scalable gateway to openstackRouting
- Modified openstack_routing to use scalable gateway application - Add GatewayNode Builder method Change-Id: Ib12d52009eeb89ab59f01cafe794894aac80985e
Kyuhwi Choi · Gerrit Code Review
Commit 92d9ea44062501497617377ec722f414ec642e7b 92d9ea44 1 parent b4bf7c89
Showing 10 changed files with 216 additions and 98 deletions
apps/openstacknetworking/openstackrouting/BUCK
apps/openstacknetworking/openstackrouting/pom.xml
apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java
apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
apps/scalablegateway/network-cfg.json
apps/scalablegateway/src/main/java/org/onosproject/scalablegateway/api/GatewayNode.java
apps/scalablegateway/src/main/java/org/onosproject/scalablegateway/api/GatewayNodeConfig.java
--- a/apps/openstacknetworking/openstackrouting/BUCK
View file @92d9ea4
+++ b/apps/openstacknetworking/openstackrouting/BUCK
View file @92d9ea4
@@ -3,6 +3,7 @@ COMPILE_DEPS = [
     '//core/store/serializers:onos-core-serializers',
     '//apps/openstackinterface/api:onos-apps-openstackinterface-api',
     '//apps/openstacknetworking/api:onos-apps-openstacknetworking-api',
+    '//apps/scalablegateway:onos-apps-scalablegateway',
 ]
 BUNDLES = [
--- a/apps/openstacknetworking/openstackrouting/pom.xml
View file @92d9ea4
+++ b/apps/openstacknetworking/openstackrouting/pom.xml
View file @92d9ea4
@@ -52,6 +52,11 @@
             <artifactId>onos-app-openstackinterface-api</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-scalablegateway</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
 </project>
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
View file @92d9ea4
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
View file @92d9ea4
@@ -40,6 +40,7 @@ import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
 import org.onosproject.openstacknetworking.OpenstackPortInfo;
 import org.onosproject.openstacknetworking.OpenstackSwitchingService;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.slf4j.Logger;
 import java.nio.ByteBuffer;
@@ -62,13 +63,14 @@ public class OpenstackIcmpHandler {
     private final Map<String, OpenstackPortInfo> icmpInfoMap = Maps.newHashMap();
     private final OpenstackSwitchingService openstackSwitchingService;
     private final OpenstackInterfaceService openstackService;
+    private final ScalableGatewayService gatewayService;
     private final OpenstackNetworkingConfig config;
     private static final MacAddress GATEWAY_MAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f");
     private static final String NETWORK_ROUTER_INTERFACE = "network:router_interface";
     private static final String PORTNAME = "portName";
     private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
     private static final String NETWORK_FLOATING_IP = "network:floatingip";
-
+    private static final String EXTERNAL_NODE_NULL = "There is no external node about this deviceId []";
     /**
      * Default constructor.
      *
@@ -77,15 +79,17 @@ public class OpenstackIcmpHandler {
      * @param openstackService          openstackInterface service
      * @param config                    openstackRoutingConfig
      * @param openstackSwitchingService openstackSwitching service
+     * @param gatewayService scalable gateway service
      */
     OpenstackIcmpHandler(PacketService packetService, DeviceService deviceService,
                          OpenstackInterfaceService openstackService, OpenstackNetworkingConfig config,
-                         OpenstackSwitchingService openstackSwitchingService) {
+                         OpenstackSwitchingService openstackSwitchingService, ScalableGatewayService gatewayService) {
         this.packetService = packetService;
         this.deviceService = deviceService;
         this.openstackService = checkNotNull(openstackService);
         this.config = checkNotNull(config);
         this.openstackSwitchingService = checkNotNull(openstackSwitchingService);
+        this.gatewayService = gatewayService;
     }
     /**
@@ -99,10 +103,13 @@ public class OpenstackIcmpHandler {
                 .matchIPProtocol(IPv4.PROTOCOL_ICMP)
                 .build();
+        Map<DeviceId, PortNumber> externalInfoMap = getExternalInfo();
+
+        externalInfoMap.keySet().forEach(deviceId ->
                 packetService.requestPackets(icmpSelector,
                         PacketPriority.CONTROL,
                         appId,
-                Optional.of(DeviceId.deviceId(config.gatewayBridgeId())));
+                        Optional.of(deviceId)));
     }
     /**
@@ -223,14 +230,19 @@ public class OpenstackIcmpHandler {
         Ethernet icmpRequestEth = new Ethernet();
         icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4)
-                .setSourceMACAddress(MacAddress.valueOf(config.gatewayExternalInterfaceMac()))
-                .setDestinationMACAddress(MacAddress.valueOf(config.physicalRouterMac()))
                 .setPayload(icmpRequestIpv4);
-        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-                .setOutput(getPortForAnnotationPortName(DeviceId.deviceId(config.gatewayBridgeId()),
+
-                        config.gatewayExternalInterfaceName()))
+        Map<DeviceId, PortNumber> externalInforMap = getExternalInfo();
-                .build();
+
+        if (externalInforMap.size() == 0 || !externalInforMap.containsKey(deviceId)) {
+            log.error(EXTERNAL_NODE_NULL, deviceId.toString());
+            return;
+        }
+        tBuilder.setOutput(externalInforMap.get(deviceId));
+
+        TrafficTreatment treatment = tBuilder.build();
         OutboundPacket packet = new DefaultOutboundPacket(deviceId,
                 treatment, ByteBuffer.wrap(icmpRequestEth.serialize()));
@@ -323,4 +335,10 @@ public class OpenstackIcmpHandler {
         }
         return true;
     }
+    private Map<DeviceId, PortNumber> getExternalInfo() {
+        Map<DeviceId, PortNumber> externalInfoMap = Maps.newHashMap();
+        gatewayService.getGatewayDeviceIds().forEach(deviceId ->
+                externalInfoMap.putIfAbsent(deviceId, gatewayService.getGatewayExternalPorts(deviceId).get(0)));
+        return externalInfoMap;
+    }
 }
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
View file @92d9ea4
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
View file @92d9ea4
@@ -30,10 +30,11 @@ import org.onosproject.net.packet.InboundPacket;
 import org.onosproject.net.packet.PacketContext;
 import org.onosproject.net.packet.PacketService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
-import org.onosproject.openstackinterface.OpenstackNetwork;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstackinterface.OpenstackRouter;
 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -60,6 +61,7 @@ public class OpenstackPnatHandler implements Runnable {
     private OpenstackNetworkingConfig config;
     private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
+    private static final String EXTERNAL_PORT_NULL = "There is no external port in this deviceId []";
     OpenstackPnatHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context,
                          int portNum, OpenstackPort openstackPort, Port port, OpenstackNetworkingConfig config) {
@@ -84,16 +86,17 @@ public class OpenstackPnatHandler implements Runnable {
         OpenstackRouter router = getOpenstackRouter(openstackPort);
+        MacAddress externalMac = MacAddress.NONE;
+        MacAddress routerMac = MacAddress.NONE;
+
         rulePopulator.populatePnatFlowRules(inboundPacket, openstackPort, portNum,
-                getExternalIp(router), MacAddress.valueOf(config.gatewayExternalInterfaceMac()),
+                getExternalIp(router), externalMac, routerMac);
-                MacAddress.valueOf(config.physicalRouterMac()));
         packetOut((Ethernet) ethernet.clone(), inboundPacket.receivedFrom().deviceId(), portNum, router);
     }
     private OpenstackRouter getOpenstackRouter(OpenstackPort openstackPort) {
         OpenstackInterfaceService networkingService = getService(OpenstackInterfaceService.class);
-        OpenstackNetwork network = networkingService.network(openstackPort.networkId());
         OpenstackPort port = networkingService.ports()
                 .stream()
@@ -146,11 +149,16 @@ public class OpenstackPnatHandler implements Runnable {
         iPacket.resetChecksum();
         iPacket.setParent(ethernet);
         ethernet.setPayload(iPacket);
-        ethernet.setSourceMACAddress(config.gatewayExternalInterfaceMac())
+        ScalableGatewayService gatewayService = getService(ScalableGatewayService.class);
-                .setDestinationMACAddress(config.physicalRouterMac());
+        GatewayNode gatewayNode = gatewayService.getGatewayNode(deviceId);
+        if (gatewayNode.getGatewayExternalInterfaceNames().size() == 0) {
+            log.error(EXTERNAL_PORT_NULL, deviceId.toString());
+            return;
+        }
+        treatment.setOutput(gatewayService.getGatewayExternalPorts(deviceId).get(0));
+
         ethernet.resetChecksum();
-        treatment.setOutput(port.number());
         packetService.emit(new DefaultOutboundPacket(deviceId, treatment.build(),
                 ByteBuffer.wrap(ethernet.serialize())));
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java
View file @92d9ea4
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java
View file @92d9ea4
@@ -15,6 +15,7 @@
  */
 package org.onosproject.openstacknetworking.routing;
+import com.google.common.collect.Lists;
 import org.onlab.packet.ARP;
 import org.onlab.packet.EthType;
 import org.onlab.packet.Ethernet;
@@ -34,9 +35,11 @@ import org.onosproject.net.packet.PacketService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.slf4j.Logger;
 import java.nio.ByteBuffer;
+import java.util.List;
 import java.util.Optional;
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -51,21 +54,23 @@ public class OpenstackRoutingArpHandler {
     private final PacketService packetService;
     private final OpenstackInterfaceService openstackService;
     private final OpenstackNetworkingConfig config;
+    private final ScalableGatewayService gatewayService;
     private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
     private static final String NETWORK_FLOATING_IP = "network:floatingip";
     /**
      * Default constructor.
-     *
      *  @param packetService packet service
      * @param openstackService openstackInterface service
      * @param config openstackRoutingConfig
+     * @param gatewayService
      */
     OpenstackRoutingArpHandler(PacketService packetService, OpenstackInterfaceService openstackService,
-                               OpenstackNetworkingConfig config) {
+                               OpenstackNetworkingConfig config, ScalableGatewayService gatewayService) {
         this.packetService = packetService;
         this.openstackService = checkNotNull(openstackService);
         this.config = checkNotNull(config);
+        this.gatewayService = gatewayService;
     }
     /**
@@ -79,10 +84,12 @@ public class OpenstackRoutingArpHandler {
                 .matchEthType(EthType.EtherType.ARP.ethType().toShort())
                 .build();
+        getExternalInfo().forEach(deviceId ->
                 packetService.requestPackets(arpSelector,
                         PacketPriority.CONTROL,
                         appId,
-                Optional.of(DeviceId.deviceId(config.gatewayBridgeId())));
+                        Optional.of(deviceId))
+        );
     }
     /**
@@ -138,4 +145,10 @@ public class OpenstackRoutingArpHandler {
         }
         return port.macAddress();
     }
+
+    private List<DeviceId> getExternalInfo() {
+        List<DeviceId> externalInfoList = Lists.newArrayList();
+        gatewayService.getGatewayDeviceIds().forEach(externalInfoList::add);
+        return externalInfoList;
+    }
 }
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
View file @92d9ea4
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
View file @92d9ea4
@@ -56,6 +56,7 @@ import org.onosproject.openstacknetworking.OpenstackPortInfo;
 import org.onosproject.openstacknetworking.OpenstackRoutingService;
 import org.onosproject.openstacknetworking.OpenstackSubjectFactories;
 import org.onosproject.openstacknetworking.OpenstackSwitchingService;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.onosproject.store.serializers.KryoNamespaces;
 import org.onosproject.store.service.ConsistentMap;
 import org.onosproject.store.service.Serializer;
@@ -113,6 +114,8 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected StorageService storageService;
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ScalableGatewayService gatewayService;
     private ApplicationId appId;
     private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
@@ -444,8 +447,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
             if (context.isHandled()) {
                 return;
-            } else if (!context.inPacket().receivedFrom().deviceId().toString()
+            } else if (!checkGatewayNode(context.inPacket().receivedFrom().deviceId())) {
-                    .equals(config.gatewayBridgeId())) {
                 return;
             }
@@ -472,16 +474,19 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
                         }
                     default:
                         int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
-                        Optional<Port> port =
+                        DeviceId deviceId = pkt.receivedFrom().deviceId();
-                                getExternalPort(pkt.receivedFrom().deviceId(), config.gatewayExternalInterfaceName());
+                        Port port = null;
-                        if (port.isPresent()) {
+                        port = deviceService.getPort(deviceId,
+                                gatewayService.getGatewayExternalPorts(deviceId).get(0));
+                        if (port != null) {
                             OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
                                     Ip4Address.valueOf(iPacket.getSourceAddress()));
                             l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
-                                    portNum, openstackPort, port.get(), config));
+                                    portNum, openstackPort, port, config));
                         } else {
                             log.warn("There`s no external interface");
                         }
+
                         break;
                 }
             } else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
@@ -511,6 +516,10 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
     }
+    private boolean checkGatewayNode(DeviceId deviceId) {
+        return gatewayService.getGatewayDeviceIds().contains(deviceId);
+    }
+
     private void clearPortNumMap() {
         tpPortNumMap.entrySet().forEach(e -> {
             if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
@@ -600,21 +609,12 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
             return;
         }
-        checkNotNull(config.physicalRouterMac());
+        rulePopulator = new OpenstackRoutingRulePopulator(appId, openstackService, flowObjectiveService,
-        checkNotNull(config.gatewayBridgeId());
+                deviceService, driverService, config, gatewayService);
-        checkNotNull(config.gatewayExternalInterfaceMac());
-        checkNotNull(config.gatewayExternalInterfaceName());
-
-        log.warn("Configured info: {}, {}, {}, {}", config.physicalRouterMac(), config.gatewayBridgeId(),
-                config.gatewayExternalInterfaceMac(), config.gatewayExternalInterfaceName());
-
-        rulePopulator = new OpenstackRoutingRulePopulator(appId,
-                openstackService, flowObjectiveService, deviceService, driverService, config);
-
         openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService,
-                openstackService, config, openstackSwitchingService);
+                openstackService, config, openstackSwitchingService, gatewayService);
-        openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService, config);
+        openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService,
-
+                config, gatewayService);
         openstackIcmpHandler.requestPacket(appId);
         openstackArpHandler.requestPacket(appId);
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
View file @92d9ea4
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
View file @92d9ea4
@@ -15,6 +15,7 @@
  */
 package org.onosproject.openstacknetworking.routing;
+import com.google.common.collect.Lists;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.IPv4;
 import org.onlab.packet.Ip4Address;
@@ -25,6 +26,7 @@ import org.onlab.packet.TCP;
 import org.onlab.packet.TpPort;
 import org.onlab.packet.UDP;
 import org.onosproject.core.ApplicationId;
+import org.onosproject.core.GroupId;
 import org.onosproject.net.Device;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.Port;
@@ -55,6 +57,7 @@ import org.onosproject.openstackinterface.OpenstackFloatingIP;
 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
 import org.onosproject.openstacknetworking.OpenstackPortInfo;
 import org.onosproject.openstacknetworking.OpenstackRoutingService;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -77,6 +80,7 @@ public class OpenstackRoutingRulePopulator {
     private final DeviceService deviceService;
     private final DriverService driverService;
     private final OpenstackNetworkingConfig config;
+    private final ScalableGatewayService gatewayService;
     private static final String PORTNAME_PREFIX_TUNNEL = "vxlan";
     private static final String PORTNAME = "portName";
@@ -84,6 +88,7 @@ public class OpenstackRoutingRulePopulator {
     private static final String PORTNOTNULL = "Port can not be null";
     private static final String DEVICENOTNULL = "Device can not be null";
+    private static final String EXTPORTNOTNULL = "External port can not be null";
     private static final String TUNNEL_DESTINATION = "tunnelDst";
     private static final int ROUTING_RULE_PRIORITY = 25000;
     private static final int FLOATING_RULE_PRIORITY = 42000;
@@ -107,16 +112,19 @@ public class OpenstackRoutingRulePopulator {
      * @param deviceService DeviceService
      * @param driverService DriverService
      * @param config Configuration for openstack environment
+     * @param gatewayService scalable gateway service
      */
     public OpenstackRoutingRulePopulator(ApplicationId appId, OpenstackInterfaceService openstackService,
                                          FlowObjectiveService flowObjectiveService, DeviceService deviceService,
-                                         DriverService driverService, OpenstackNetworkingConfig config) {
+                                         DriverService driverService, OpenstackNetworkingConfig config,
+                                         ScalableGatewayService gatewayService) {
         this.appId = appId;
         this.flowObjectiveService = flowObjectiveService;
         this.openstackService = checkNotNull(openstackService);
         this.deviceService = deviceService;
         this.driverService = driverService;
         this.config = config;
+        this.gatewayService = gatewayService;
     }
     /**
@@ -155,9 +163,6 @@ public class OpenstackRoutingRulePopulator {
                 .matchIPDst(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
         TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-        tBuilder.setEthSrc(externalInterface)
-                .setEthDst(externalRouter)
-                .setIpSrc(externalIp);
         switch (iPacket.getProtocol()) {
             case IPv4.PROTOCOL_TCP:
@@ -177,9 +182,9 @@ public class OpenstackRoutingRulePopulator {
                 break;
         }
-        Port port = checkNotNull(getPortOfExternalInterface(), PORTNOTNULL);
+        tBuilder.setIpSrc(externalIp);
-        tBuilder.setOutput(port.number());
+        gatewayService.getGatewayNodes().forEach(node -> {
-
+            tBuilder.setOutput(gatewayService.getGatewayExternalPorts(node.getGatewayDeviceId()).get(0));
             ForwardingObjective fo = DefaultForwardingObjective.builder()
                     .withSelector(sBuilder.build())
                     .withTreatment(tBuilder.build())
@@ -189,7 +194,8 @@ public class OpenstackRoutingRulePopulator {
                     .fromApp(appId)
                     .add();
-        flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo);
+            flowObjectiveService.forward(node.getGatewayDeviceId(), fo);
+        });
     }
@@ -202,7 +208,6 @@ public class OpenstackRoutingRulePopulator {
     private void populatePnatIncomingFlowRules(long vni, Ip4Address externalIp) {
         IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
-        DeviceId deviceId = inboundPacket.receivedFrom().deviceId();
         TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
         sBuilder.matchEthType(Ethernet.TYPE_IPV4)
@@ -232,6 +237,8 @@ public class OpenstackRoutingRulePopulator {
                 break;
         }
+        getGatewayNodeList().forEach(node -> {
+            DeviceId deviceId = node.id();
             tBuilder.extension(buildNiciraExtenstion(deviceId, getHostIpfromOpenstackPort(openstackPort)), deviceId)
                     .setOutput(getTunnelPort(deviceId));
@@ -244,7 +251,15 @@ public class OpenstackRoutingRulePopulator {
                     .fromApp(appId)
                     .add();
-        flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo);
+            flowObjectiveService.forward(deviceId, fo);
+        });
+    }
+
+    private List<Device> getGatewayNodeList() {
+        List<Device> devices = Lists.newArrayList();
+        gatewayService.getGatewayDeviceIds().forEach(deviceId ->
+                devices.add(checkNotNull(deviceService.getDevice(deviceId))));
+        return devices;
     }
     private Ip4Address getHostIpfromOpenstackPort(OpenstackPort openstackPort) {
@@ -255,15 +270,15 @@ public class OpenstackRoutingRulePopulator {
     private Device getDevicefromOpenstackPort(OpenstackPort openstackPort) {
         String openstackPortName = PORTNAME_PREFIX_VM + openstackPort.id().substring(0, 11);
         Device device = StreamSupport.stream(deviceService.getDevices().spliterator(), false)
-                .filter(d -> findPortinDevice(d, openstackPortName))
+                .filter(d -> findPortinDevice(d.id(), openstackPortName))
                 .iterator()
                 .next();
         checkNotNull(device, DEVICENOTNULL);
         return device;
     }
-    private boolean findPortinDevice(Device d, String openstackPortName) {
+    private boolean findPortinDevice(DeviceId deviceId, String openstackPortName) {
-        Port port = deviceService.getPorts(d.id())
+        Port port = deviceService.getPorts(deviceId)
                 .stream()
                 .filter(p -> p.isEnabled() && p.annotations().value(PORTNAME).equals(openstackPortName))
                 .findAny()
@@ -274,13 +289,13 @@ public class OpenstackRoutingRulePopulator {
     /**
      * Builds Nicira extension for tagging remoteIp of vxlan.
      *
-     * @param id Device Id of vxlan source device
+     * @param deviceId Device Id of vxlan source device
      * @param hostIp Remote Ip of vxlan destination device
      * @return NiciraExtension Treatment
      */
-    public ExtensionTreatment buildNiciraExtenstion(DeviceId id, Ip4Address hostIp) {
+    public ExtensionTreatment buildNiciraExtenstion(DeviceId deviceId, Ip4Address hostIp) {
-        Driver driver = driverService.getDriver(id);
+        Driver driver = driverService.getDriver(deviceId);
-        DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, id));
+        DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, deviceId));
         ExtensionTreatmentResolver resolver = driverHandler.behaviour(ExtensionTreatmentResolver.class);
         ExtensionTreatment extensionInstruction =
@@ -329,7 +344,6 @@ public class OpenstackRoutingRulePopulator {
     }
     private void populateRuleGatewaytoController(long vni) {
-        Device gatewayDevice = getGatewayNode();
         TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
         TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
@@ -346,26 +360,45 @@ public class OpenstackRoutingRulePopulator {
                 .fromApp(appId)
                 .add();
-        flowObjectiveService.forward(gatewayDevice.id(), fo);
+        getGatewayNodeList().forEach(device -> flowObjectiveService.forward(device.id(), fo));
     }
     private void populateComputeNodeRules(long vni) {
-        Device gatewayDevice = getGatewayNode();
-
         StreamSupport.stream(deviceService.getDevices().spliterator(), false)
                 .filter(d -> !checkGatewayNode(d.id()))
-                .forEach(d -> populateRuleToGateway(d, gatewayDevice, vni));
+                .forEach(d -> populateRuleToGatewayBySgw(d.id(),
+                        gatewayService.getGroupIdForGatewayLoadBalance(d.id()), vni));
+    }
+
+    private void populateRuleToGatewayBySgw(DeviceId deviceId, GroupId groupId, long vni) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(vni)
+                .matchEthDst(GATEWAYMAC);
+        tBuilder.group(groupId);
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .withPriority(ROUTING_RULE_PRIORITY)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(deviceId, fo);
     }
-    private void populateRuleToGateway(Device d, Device gatewayDevice, long vni) {
+    private void populateRuleToGateway(DeviceId deviceId, Device gatewayDevice, long vni) {
         TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
         TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
         sBuilder.matchEthType(Ethernet.TYPE_IPV4)
                 .matchTunnelId(vni)
                 .matchEthDst(GATEWAYMAC);
-        tBuilder.extension(buildNiciraExtenstion(d.id(), config.nodes().get(gatewayDevice.id())), d.id())
+        tBuilder.extension(buildNiciraExtenstion(deviceId, config.nodes().get(gatewayDevice.id())), deviceId)
-                .setOutput(getTunnelPort(d.id()));
+                .setOutput(getTunnelPort(deviceId));
         ForwardingObjective fo = DefaultForwardingObjective.builder()
                 .withSelector(sBuilder.build())
@@ -375,7 +408,7 @@ public class OpenstackRoutingRulePopulator {
                 .fromApp(appId)
                 .add();
-        flowObjectiveService.forward(d.id(), fo);
+        flowObjectiveService.forward(deviceId, fo);
     }
     private Device getGatewayNode() {
@@ -383,7 +416,7 @@ public class OpenstackRoutingRulePopulator {
     }
     private boolean checkGatewayNode(DeviceId deviceId) {
-        return deviceId.toString().equals(config.gatewayBridgeId());
+        return gatewayService.getGatewayDeviceIds().stream().anyMatch(dId -> dId.equals(deviceId));
     }
     private long getVni(String netId) {
@@ -412,7 +445,7 @@ public class OpenstackRoutingRulePopulator {
     }
-    private void removeRule(DeviceId id, TrafficSelector.Builder sBuilder,
+    private void removeRule(DeviceId deviceId, TrafficSelector.Builder sBuilder,
                             ForwardingObjective.Flag flag, int priority) {
         TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
@@ -424,7 +457,7 @@ public class OpenstackRoutingRulePopulator {
                 .fromApp(appId)
                 .remove();
-        flowObjectiveService.forward(id, fo);
+        flowObjectiveService.forward(deviceId, fo);
     }
     /**
@@ -442,8 +475,6 @@ public class OpenstackRoutingRulePopulator {
     private void populateFloatingIpIncomingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
         DeviceId portDeviceId = getDevicefromOpenstackPort(port).id();
-        Device gatewayNode = getGatewayNode();
-        Device portNode = deviceService.getDevice(portDeviceId);
         TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
         TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
@@ -451,13 +482,14 @@ public class OpenstackRoutingRulePopulator {
         sBuilder.matchEthType(Ethernet.TYPE_IPV4)
                 .matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
+        getGatewayNodeList().forEach(device -> {
+            DeviceId deviceId = device.id();
             tBuilder.setEthSrc(GATEWAYMAC)
                     .setEthDst(port.macAddress())
                     .setIpDst(floatingIP.fixedIpAddress())
                     .setTunnelId(getVni(port.networkId()))
-                .extension(buildNiciraExtenstion(gatewayNode.id(),
+                    .extension(buildNiciraExtenstion(deviceId, config.nodes().get(portDeviceId)), deviceId)
-                        config.nodes().get(portNode.id())), gatewayNode.id())
+                    .setOutput(getTunnelPort(deviceId));
-                .setOutput(getTunnelPort(gatewayNode.id()));
             ForwardingObjective fo = DefaultForwardingObjective.builder()
                     .withSelector(sBuilder.build())
@@ -467,13 +499,11 @@ public class OpenstackRoutingRulePopulator {
                     .fromApp(appId)
                     .add();
-        flowObjectiveService.forward(getGatewayNode().id(), fo);
+            flowObjectiveService.forward(deviceId, fo);
-
+        });
     }
     private void populateFloatingIpOutgoingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
-        Port outputPort = checkNotNull(getPortOfExternalInterface(), PORTNOTNULL);
-
         TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
         TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
@@ -481,10 +511,10 @@ public class OpenstackRoutingRulePopulator {
                 .matchTunnelId(getVni(port.networkId()))
                 .matchIPSrc(IpPrefix.valueOf(floatingIP.fixedIpAddress(), 32));
+        getGatewayNodeList().forEach(device -> {
+            DeviceId deviceId = device.id();
             tBuilder.setIpSrc(floatingIP.floatingIpAddress())
-                .setEthSrc(MacAddress.valueOf(config.gatewayExternalInterfaceMac()))
+                    .setOutput(getExternalPortNum(deviceId));
-                .setEthDst(MacAddress.valueOf(config.physicalRouterMac()))
-                .setOutput(outputPort.number());
             ForwardingObjective fo = DefaultForwardingObjective.builder()
                     .withSelector(sBuilder.build())
@@ -494,7 +524,12 @@ public class OpenstackRoutingRulePopulator {
                     .fromApp(appId)
                     .add();
-        flowObjectiveService.forward(getGatewayNode().id(), fo);
+            flowObjectiveService.forward(deviceId, fo);
+        });
+    }
+
+    private PortNumber getExternalPortNum(DeviceId deviceId) {
+        return checkNotNull(gatewayService.getGatewayExternalPorts(deviceId).get(0), PORTNOTNULL);
     }
     /**
@@ -514,38 +549,39 @@ public class OpenstackRoutingRulePopulator {
         sIncomingBuilder.matchEthType(Ethernet.TYPE_IPV4)
                 .matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
-        removeRule(getGatewayNode().id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
+        getGatewayNodeList().forEach(device -> {
-        removeRule(getGatewayNode().id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
+            removeRule(device.id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
+            removeRule(device.id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
+        });
     }
     /**
      * Populates L3 rules for east to west traffic.
      *
-     * @param p target VM
+     * @param openstackPort target VM
      * @param targetList target openstackRouterInterfaces
      */
-    public void populateL3Rules(OpenstackPort p, List<OpenstackRouterInterface> targetList) {
+    public void populateL3Rules(OpenstackPort openstackPort, List<OpenstackRouterInterface> targetList) {
-        Device device = getDevicefromOpenstackPort(p);
+        Device device = getDevicefromOpenstackPort(openstackPort);
-        Port port = getPortFromOpenstackPort(device, p);
+        Port port = getPortFromOpenstackPort(device, openstackPort);
-        Ip4Address vmIp = (Ip4Address) p.fixedIps().values().iterator().next();
+        Ip4Address vmIp = openstackPort.fixedIps().values().iterator().next();
         if (port == null) {
             return;
         }
         targetList.forEach(routerInterface -> {
-            OpenstackPort openstackPort = openstackService.port(routerInterface.portId());
+            long vni = getVni(openstackService.port(routerInterface.portId()).networkId());
-            long vni = getVni(openstackPort.networkId());
             if (vmIp == null) {
                 return;
             }
-            populateL3RulestoSameNode(vmIp, p, port, device, vni);
+            populateL3RulestoSameNode(vmIp, openstackPort, port, device, vni);
             deviceService.getAvailableDevices().forEach(d -> {
-                if (!d.equals(device) && !d.equals(getGatewayNode())) {
+                if (!d.equals(device) && !checkGatewayNode(d.id())) {
-                    populateL3RulestoDifferentNode(vmIp, vni, d.id(), getHostIpfromOpenstackPort(p));
+                    populateL3RulestoDifferentNode(vmIp, vni, d.id(), getHostIpfromOpenstackPort(openstackPort));
                 }
             });
@@ -618,7 +654,7 @@ public class OpenstackRoutingRulePopulator {
         OpenstackRoutingService routingService = getService(OpenstackRoutingService.class);
         deviceService.getAvailableDevices().forEach(d -> {
-            if (!d.equals(getGatewayNode())) {
+            if (!checkGatewayNode(d.id())) {
                 routerInterfaces.forEach(routerInterface -> {
                     String networkId = routingService.networkIdForRouterInterface(routerInterface.portId());
                     long vni = getVni(networkId);
--- a/apps/scalablegateway/network-cfg.json 0 → 100644
View file @92d9ea4
+++ b/apps/scalablegateway/network-cfg.json 0 → 100644
View file @92d9ea4
+{
+    "apps" : {
+        "org.onosproject.scalablegateway" : {
+            "scalablegateway" : {
+              "nodes" : [
+              {
+                "bridgeId" : "of:0000000000000003",
+                "dataPlaneIp" : "10.62.148.17",
+                "gatewayExternalInterfaceName" : "veth0"
+                }]
+            }
+        }
+    }
+}
--- a/apps/scalablegateway/src/main/java/org/onosproject/scalablegateway/api/GatewayNode.java
View file @92d9ea4
+++ b/apps/scalablegateway/src/main/java/org/onosproject/scalablegateway/api/GatewayNode.java
View file @92d9ea4
@@ -66,6 +66,15 @@ public final class GatewayNode {
     }
     /**
+     * Returns GatewayNode builder object.
+     *
+     * @return GatewayNode builder
+     */
+    public static GatewayNode.Builder builder() {
+        return new Builder();
+    }
+
+    /**
      * GatewayNode Builder class.
      */
     public static final class Builder {
--- a/apps/scalablegateway/src/main/java/org/onosproject/scalablegateway/api/GatewayNodeConfig.java
View file @92d9ea4
+++ b/apps/scalablegateway/src/main/java/org/onosproject/scalablegateway/api/GatewayNodeConfig.java
View file @92d9ea4
@@ -17,6 +17,7 @@ package org.onosproject.scalablegateway.api;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import org.onlab.packet.Ip4Address;
@@ -28,9 +29,10 @@ import org.slf4j.Logger;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.StreamSupport;
-import static org.slf4j.LoggerFactory.getLogger;
 import static org.onosproject.net.config.Config.FieldPresence.MANDATORY;
+import static org.slf4j.LoggerFactory.getLogger;
 /**
  * Configuration object for OpensatckNode service.
@@ -79,9 +81,21 @@ public class GatewayNodeConfig extends Config<ApplicationId> {
     @Override
     public boolean isValid() {
-        return hasOnlyFields(NODES, BRIDGE_ID, DATAPLANE_IP, EXTERNAL_INTERFACE_NAME) &&
+        JsonNode jsonNodes = object.get(NODES);
-                isIpAddress(DATAPLANE_IP, MANDATORY) && isString(BRIDGE_ID, MANDATORY) &&
+
-                isString(EXTERNAL_INTERFACE_NAME, MANDATORY);
+        if (jsonNodes == null) {
+            return false;
+        }
+
+        return hasOnlyFields(NODES)
+                && StreamSupport.stream(jsonNodes.spliterator(), false).allMatch(this::checkValid);
+    }
+
+    private boolean checkValid(JsonNode jsonNode) {
+        ObjectNode objectNode = (ObjectNode) jsonNode;
+        return isString(objectNode, BRIDGE_ID, MANDATORY)
+                && isIpAddress(objectNode, DATAPLANE_IP, MANDATORY)
+                && isString(objectNode, EXTERNAL_INTERFACE_NAME, MANDATORY);
     }
 }