Kyuhwi Choi
Committed by Gerrit Code Review

[ONOS-4481] Integrate scalable gateway to openstackRouting

 - Modified openstack_routing to use scalable gateway application
 - Add GatewayNode Builder method

Change-Id: Ib12d52009eeb89ab59f01cafe794894aac80985e
......@@ -3,6 +3,7 @@ COMPILE_DEPS = [
'//core/store/serializers:onos-core-serializers',
'//apps/openstackinterface/api:onos-apps-openstackinterface-api',
'//apps/openstacknetworking/api:onos-apps-openstacknetworking-api',
'//apps/scalablegateway:onos-apps-scalablegateway',
]
BUNDLES = [
......
......@@ -52,6 +52,11 @@
<artifactId>onos-app-openstackinterface-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-scalablegateway</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>
......
......@@ -40,6 +40,7 @@ import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
import org.onosproject.openstacknetworking.OpenstackPortInfo;
import org.onosproject.openstacknetworking.OpenstackSwitchingService;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import java.nio.ByteBuffer;
......@@ -62,13 +63,14 @@ public class OpenstackIcmpHandler {
private final Map<String, OpenstackPortInfo> icmpInfoMap = Maps.newHashMap();
private final OpenstackSwitchingService openstackSwitchingService;
private final OpenstackInterfaceService openstackService;
private final ScalableGatewayService gatewayService;
private final OpenstackNetworkingConfig config;
private static final MacAddress GATEWAY_MAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f");
private static final String NETWORK_ROUTER_INTERFACE = "network:router_interface";
private static final String PORTNAME = "portName";
private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
private static final String NETWORK_FLOATING_IP = "network:floatingip";
private static final String EXTERNAL_NODE_NULL = "There is no external node about this deviceId []";
/**
* Default constructor.
*
......@@ -77,15 +79,17 @@ public class OpenstackIcmpHandler {
* @param openstackService openstackInterface service
* @param config openstackRoutingConfig
* @param openstackSwitchingService openstackSwitching service
* @param gatewayService scalable gateway service
*/
OpenstackIcmpHandler(PacketService packetService, DeviceService deviceService,
OpenstackInterfaceService openstackService, OpenstackNetworkingConfig config,
OpenstackSwitchingService openstackSwitchingService) {
OpenstackSwitchingService openstackSwitchingService, ScalableGatewayService gatewayService) {
this.packetService = packetService;
this.deviceService = deviceService;
this.openstackService = checkNotNull(openstackService);
this.config = checkNotNull(config);
this.openstackSwitchingService = checkNotNull(openstackSwitchingService);
this.gatewayService = gatewayService;
}
/**
......@@ -99,10 +103,13 @@ public class OpenstackIcmpHandler {
.matchIPProtocol(IPv4.PROTOCOL_ICMP)
.build();
Map<DeviceId, PortNumber> externalInfoMap = getExternalInfo();
externalInfoMap.keySet().forEach(deviceId ->
packetService.requestPackets(icmpSelector,
PacketPriority.CONTROL,
appId,
Optional.of(DeviceId.deviceId(config.gatewayBridgeId())));
Optional.of(deviceId)));
}
/**
......@@ -223,14 +230,19 @@ public class OpenstackIcmpHandler {
Ethernet icmpRequestEth = new Ethernet();
icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4)
.setSourceMACAddress(MacAddress.valueOf(config.gatewayExternalInterfaceMac()))
.setDestinationMACAddress(MacAddress.valueOf(config.physicalRouterMac()))
.setPayload(icmpRequestIpv4);
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
.setOutput(getPortForAnnotationPortName(DeviceId.deviceId(config.gatewayBridgeId()),
config.gatewayExternalInterfaceName()))
.build();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
Map<DeviceId, PortNumber> externalInforMap = getExternalInfo();
if (externalInforMap.size() == 0 || !externalInforMap.containsKey(deviceId)) {
log.error(EXTERNAL_NODE_NULL, deviceId.toString());
return;
}
tBuilder.setOutput(externalInforMap.get(deviceId));
TrafficTreatment treatment = tBuilder.build();
OutboundPacket packet = new DefaultOutboundPacket(deviceId,
treatment, ByteBuffer.wrap(icmpRequestEth.serialize()));
......@@ -323,4 +335,10 @@ public class OpenstackIcmpHandler {
}
return true;
}
private Map<DeviceId, PortNumber> getExternalInfo() {
Map<DeviceId, PortNumber> externalInfoMap = Maps.newHashMap();
gatewayService.getGatewayDeviceIds().forEach(deviceId ->
externalInfoMap.putIfAbsent(deviceId, gatewayService.getGatewayExternalPorts(deviceId).get(0)));
return externalInfoMap;
}
}
......
......@@ -30,10 +30,11 @@ import org.onosproject.net.packet.InboundPacket;
import org.onosproject.net.packet.PacketContext;
import org.onosproject.net.packet.PacketService;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
import org.onosproject.openstackinterface.OpenstackNetwork;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstackinterface.OpenstackRouter;
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
import org.onosproject.scalablegateway.api.GatewayNode;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -60,6 +61,7 @@ public class OpenstackPnatHandler implements Runnable {
private OpenstackNetworkingConfig config;
private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
private static final String EXTERNAL_PORT_NULL = "There is no external port in this deviceId []";
OpenstackPnatHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context,
int portNum, OpenstackPort openstackPort, Port port, OpenstackNetworkingConfig config) {
......@@ -84,16 +86,17 @@ public class OpenstackPnatHandler implements Runnable {
OpenstackRouter router = getOpenstackRouter(openstackPort);
MacAddress externalMac = MacAddress.NONE;
MacAddress routerMac = MacAddress.NONE;
rulePopulator.populatePnatFlowRules(inboundPacket, openstackPort, portNum,
getExternalIp(router), MacAddress.valueOf(config.gatewayExternalInterfaceMac()),
MacAddress.valueOf(config.physicalRouterMac()));
getExternalIp(router), externalMac, routerMac);
packetOut((Ethernet) ethernet.clone(), inboundPacket.receivedFrom().deviceId(), portNum, router);
}
private OpenstackRouter getOpenstackRouter(OpenstackPort openstackPort) {
OpenstackInterfaceService networkingService = getService(OpenstackInterfaceService.class);
OpenstackNetwork network = networkingService.network(openstackPort.networkId());
OpenstackPort port = networkingService.ports()
.stream()
......@@ -146,11 +149,16 @@ public class OpenstackPnatHandler implements Runnable {
iPacket.resetChecksum();
iPacket.setParent(ethernet);
ethernet.setPayload(iPacket);
ethernet.setSourceMACAddress(config.gatewayExternalInterfaceMac())
.setDestinationMACAddress(config.physicalRouterMac());
ScalableGatewayService gatewayService = getService(ScalableGatewayService.class);
GatewayNode gatewayNode = gatewayService.getGatewayNode(deviceId);
if (gatewayNode.getGatewayExternalInterfaceNames().size() == 0) {
log.error(EXTERNAL_PORT_NULL, deviceId.toString());
return;
}
treatment.setOutput(gatewayService.getGatewayExternalPorts(deviceId).get(0));
ethernet.resetChecksum();
treatment.setOutput(port.number());
packetService.emit(new DefaultOutboundPacket(deviceId, treatment.build(),
ByteBuffer.wrap(ethernet.serialize())));
......
......@@ -15,6 +15,7 @@
*/
package org.onosproject.openstacknetworking.routing;
import com.google.common.collect.Lists;
import org.onlab.packet.ARP;
import org.onlab.packet.EthType;
import org.onlab.packet.Ethernet;
......@@ -34,9 +35,11 @@ import org.onosproject.net.packet.PacketService;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Optional;
import static com.google.common.base.Preconditions.checkNotNull;
......@@ -51,21 +54,23 @@ public class OpenstackRoutingArpHandler {
private final PacketService packetService;
private final OpenstackInterfaceService openstackService;
private final OpenstackNetworkingConfig config;
private final ScalableGatewayService gatewayService;
private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
private static final String NETWORK_FLOATING_IP = "network:floatingip";
/**
* Default constructor.
*
* @param packetService packet service
* @param openstackService openstackInterface service
* @param config openstackRoutingConfig
* @param gatewayService
*/
OpenstackRoutingArpHandler(PacketService packetService, OpenstackInterfaceService openstackService,
OpenstackNetworkingConfig config) {
OpenstackNetworkingConfig config, ScalableGatewayService gatewayService) {
this.packetService = packetService;
this.openstackService = checkNotNull(openstackService);
this.config = checkNotNull(config);
this.gatewayService = gatewayService;
}
/**
......@@ -79,10 +84,12 @@ public class OpenstackRoutingArpHandler {
.matchEthType(EthType.EtherType.ARP.ethType().toShort())
.build();
getExternalInfo().forEach(deviceId ->
packetService.requestPackets(arpSelector,
PacketPriority.CONTROL,
appId,
Optional.of(DeviceId.deviceId(config.gatewayBridgeId())));
Optional.of(deviceId))
);
}
/**
......@@ -138,4 +145,10 @@ public class OpenstackRoutingArpHandler {
}
return port.macAddress();
}
private List<DeviceId> getExternalInfo() {
List<DeviceId> externalInfoList = Lists.newArrayList();
gatewayService.getGatewayDeviceIds().forEach(externalInfoList::add);
return externalInfoList;
}
}
......
......@@ -56,6 +56,7 @@ import org.onosproject.openstacknetworking.OpenstackPortInfo;
import org.onosproject.openstacknetworking.OpenstackRoutingService;
import org.onosproject.openstacknetworking.OpenstackSubjectFactories;
import org.onosproject.openstacknetworking.OpenstackSwitchingService;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.onosproject.store.serializers.KryoNamespaces;
import org.onosproject.store.service.ConsistentMap;
import org.onosproject.store.service.Serializer;
......@@ -113,6 +114,8 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected StorageService storageService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ScalableGatewayService gatewayService;
private ApplicationId appId;
private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
......@@ -444,8 +447,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
if (context.isHandled()) {
return;
} else if (!context.inPacket().receivedFrom().deviceId().toString()
.equals(config.gatewayBridgeId())) {
} else if (!checkGatewayNode(context.inPacket().receivedFrom().deviceId())) {
return;
}
......@@ -472,16 +474,19 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
}
default:
int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
Optional<Port> port =
getExternalPort(pkt.receivedFrom().deviceId(), config.gatewayExternalInterfaceName());
if (port.isPresent()) {
DeviceId deviceId = pkt.receivedFrom().deviceId();
Port port = null;
port = deviceService.getPort(deviceId,
gatewayService.getGatewayExternalPorts(deviceId).get(0));
if (port != null) {
OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
Ip4Address.valueOf(iPacket.getSourceAddress()));
l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
portNum, openstackPort, port.get(), config));
portNum, openstackPort, port, config));
} else {
log.warn("There`s no external interface");
}
break;
}
} else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
......@@ -511,6 +516,10 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
}
private boolean checkGatewayNode(DeviceId deviceId) {
return gatewayService.getGatewayDeviceIds().contains(deviceId);
}
private void clearPortNumMap() {
tpPortNumMap.entrySet().forEach(e -> {
if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
......@@ -600,21 +609,12 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
return;
}
checkNotNull(config.physicalRouterMac());
checkNotNull(config.gatewayBridgeId());
checkNotNull(config.gatewayExternalInterfaceMac());
checkNotNull(config.gatewayExternalInterfaceName());
log.warn("Configured info: {}, {}, {}, {}", config.physicalRouterMac(), config.gatewayBridgeId(),
config.gatewayExternalInterfaceMac(), config.gatewayExternalInterfaceName());
rulePopulator = new OpenstackRoutingRulePopulator(appId,
openstackService, flowObjectiveService, deviceService, driverService, config);
rulePopulator = new OpenstackRoutingRulePopulator(appId, openstackService, flowObjectiveService,
deviceService, driverService, config, gatewayService);
openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService,
openstackService, config, openstackSwitchingService);
openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService, config);
openstackService, config, openstackSwitchingService, gatewayService);
openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService,
config, gatewayService);
openstackIcmpHandler.requestPacket(appId);
openstackArpHandler.requestPacket(appId);
......
{
"apps" : {
"org.onosproject.scalablegateway" : {
"scalablegateway" : {
"nodes" : [
{
"bridgeId" : "of:0000000000000003",
"dataPlaneIp" : "10.62.148.17",
"gatewayExternalInterfaceName" : "veth0"
}]
}
}
}
}
......@@ -66,6 +66,15 @@ public final class GatewayNode {
}
/**
* Returns GatewayNode builder object.
*
* @return GatewayNode builder
*/
public static GatewayNode.Builder builder() {
return new Builder();
}
/**
* GatewayNode Builder class.
*/
public static final class Builder {
......
......@@ -17,6 +17,7 @@ package org.onosproject.scalablegateway.api;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.onlab.packet.Ip4Address;
......@@ -28,9 +29,10 @@ import org.slf4j.Logger;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.StreamSupport;
import static org.slf4j.LoggerFactory.getLogger;
import static org.onosproject.net.config.Config.FieldPresence.MANDATORY;
import static org.slf4j.LoggerFactory.getLogger;
/**
* Configuration object for OpensatckNode service.
......@@ -79,9 +81,21 @@ public class GatewayNodeConfig extends Config<ApplicationId> {
@Override
public boolean isValid() {
return hasOnlyFields(NODES, BRIDGE_ID, DATAPLANE_IP, EXTERNAL_INTERFACE_NAME) &&
isIpAddress(DATAPLANE_IP, MANDATORY) && isString(BRIDGE_ID, MANDATORY) &&
isString(EXTERNAL_INTERFACE_NAME, MANDATORY);
JsonNode jsonNodes = object.get(NODES);
if (jsonNodes == null) {
return false;
}
return hasOnlyFields(NODES)
&& StreamSupport.stream(jsonNodes.spliterator(), false).allMatch(this::checkValid);
}
private boolean checkValid(JsonNode jsonNode) {
ObjectNode objectNode = (ObjectNode) jsonNode;
return isString(objectNode, BRIDGE_ID, MANDATORY)
&& isIpAddress(objectNode, DATAPLANE_IP, MANDATORY)
&& isString(objectNode, EXTERNAL_INTERFACE_NAME, MANDATORY);
}
}
......