홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Kyuhwi Choi
Committed by Gerrit Code Review
92d9ea44 1 parent b4bf7c89

[ONOS-4481] Integrate scalable gateway to openstackRouting 

 - Modified openstack_routing to use scalable gateway application
 - Add GatewayNode Builder method

Change-Id: Ib12d52009eeb89ab59f01cafe794894aac80985e
Inline Side-by-side
Showing 10 changed files with 265 additions and 147 deletions
......@@ -3,6 +3,7 @@ COMPILE_DEPS = [
'//core/store/serializers:onos-core-serializers',
'//apps/openstackinterface/api:onos-apps-openstackinterface-api',
'//apps/openstacknetworking/api:onos-apps-openstacknetworking-api',
'//apps/scalablegateway:onos-apps-scalablegateway',
]
BUNDLES = [
......
......@@ -52,6 +52,11 @@
<artifactId>onos-app-openstackinterface-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-scalablegateway</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>
......
......@@ -40,6 +40,7 @@ import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
import org.onosproject.openstacknetworking.OpenstackPortInfo;
import org.onosproject.openstacknetworking.OpenstackSwitchingService;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import java.nio.ByteBuffer;
......@@ -62,13 +63,14 @@ public class OpenstackIcmpHandler {
private final Map<String, OpenstackPortInfo> icmpInfoMap = Maps.newHashMap();
private final OpenstackSwitchingService openstackSwitchingService;
private final OpenstackInterfaceService openstackService;
private final ScalableGatewayService gatewayService;
private final OpenstackNetworkingConfig config;
private static final MacAddress GATEWAY_MAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f");
private static final String NETWORK_ROUTER_INTERFACE = "network:router_interface";
private static final String PORTNAME = "portName";
private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
private static final String NETWORK_FLOATING_IP = "network:floatingip";
private static final String EXTERNAL_NODE_NULL = "There is no external node about this deviceId []";
/**
* Default constructor.
*
......@@ -77,15 +79,17 @@ public class OpenstackIcmpHandler {
* @param openstackService openstackInterface service
* @param config openstackRoutingConfig
* @param openstackSwitchingService openstackSwitching service
* @param gatewayService scalable gateway service
*/
OpenstackIcmpHandler(PacketService packetService, DeviceService deviceService,
OpenstackInterfaceService openstackService, OpenstackNetworkingConfig config,
OpenstackSwitchingService openstackSwitchingService) {
OpenstackSwitchingService openstackSwitchingService, ScalableGatewayService gatewayService) {
this.packetService = packetService;
this.deviceService = deviceService;
this.openstackService = checkNotNull(openstackService);
this.config = checkNotNull(config);
this.openstackSwitchingService = checkNotNull(openstackSwitchingService);
this.gatewayService = gatewayService;
}
/**
......@@ -99,10 +103,13 @@ public class OpenstackIcmpHandler {
.matchIPProtocol(IPv4.PROTOCOL_ICMP)
.build();
packetService.requestPackets(icmpSelector,
PacketPriority.CONTROL,
appId,
Optional.of(DeviceId.deviceId(config.gatewayBridgeId())));
Map<DeviceId, PortNumber> externalInfoMap = getExternalInfo();
externalInfoMap.keySet().forEach(deviceId ->
packetService.requestPackets(icmpSelector,
PacketPriority.CONTROL,
appId,
Optional.of(deviceId)));
}
/**
......@@ -223,14 +230,19 @@ public class OpenstackIcmpHandler {
Ethernet icmpRequestEth = new Ethernet();
icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4)
.setSourceMACAddress(MacAddress.valueOf(config.gatewayExternalInterfaceMac()))
.setDestinationMACAddress(MacAddress.valueOf(config.physicalRouterMac()))
.setPayload(icmpRequestIpv4);
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
.setOutput(getPortForAnnotationPortName(DeviceId.deviceId(config.gatewayBridgeId()),
config.gatewayExternalInterfaceName()))
.build();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
Map<DeviceId, PortNumber> externalInforMap = getExternalInfo();
if (externalInforMap.size() == 0 || !externalInforMap.containsKey(deviceId)) {
log.error(EXTERNAL_NODE_NULL, deviceId.toString());
return;
}
tBuilder.setOutput(externalInforMap.get(deviceId));
TrafficTreatment treatment = tBuilder.build();
OutboundPacket packet = new DefaultOutboundPacket(deviceId,
treatment, ByteBuffer.wrap(icmpRequestEth.serialize()));
......@@ -323,4 +335,10 @@ public class OpenstackIcmpHandler {
}
return true;
}
}
\ No newline at end of file
private Map<DeviceId, PortNumber> getExternalInfo() {
Map<DeviceId, PortNumber> externalInfoMap = Maps.newHashMap();
gatewayService.getGatewayDeviceIds().forEach(deviceId ->
externalInfoMap.putIfAbsent(deviceId, gatewayService.getGatewayExternalPorts(deviceId).get(0)));
return externalInfoMap;
}
}
......
......@@ -30,10 +30,11 @@ import org.onosproject.net.packet.InboundPacket;
import org.onosproject.net.packet.PacketContext;
import org.onosproject.net.packet.PacketService;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
import org.onosproject.openstackinterface.OpenstackNetwork;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstackinterface.OpenstackRouter;
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
import org.onosproject.scalablegateway.api.GatewayNode;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -60,6 +61,7 @@ public class OpenstackPnatHandler implements Runnable {
private OpenstackNetworkingConfig config;
private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
private static final String EXTERNAL_PORT_NULL = "There is no external port in this deviceId []";
OpenstackPnatHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context,
int portNum, OpenstackPort openstackPort, Port port, OpenstackNetworkingConfig config) {
......@@ -84,16 +86,17 @@ public class OpenstackPnatHandler implements Runnable {
OpenstackRouter router = getOpenstackRouter(openstackPort);
MacAddress externalMac = MacAddress.NONE;
MacAddress routerMac = MacAddress.NONE;
rulePopulator.populatePnatFlowRules(inboundPacket, openstackPort, portNum,
getExternalIp(router), MacAddress.valueOf(config.gatewayExternalInterfaceMac()),
MacAddress.valueOf(config.physicalRouterMac()));
getExternalIp(router), externalMac, routerMac);
packetOut((Ethernet) ethernet.clone(), inboundPacket.receivedFrom().deviceId(), portNum, router);
}
private OpenstackRouter getOpenstackRouter(OpenstackPort openstackPort) {
OpenstackInterfaceService networkingService = getService(OpenstackInterfaceService.class);
OpenstackNetwork network = networkingService.network(openstackPort.networkId());
OpenstackPort port = networkingService.ports()
.stream()
......@@ -146,11 +149,16 @@ public class OpenstackPnatHandler implements Runnable {
iPacket.resetChecksum();
iPacket.setParent(ethernet);
ethernet.setPayload(iPacket);
ethernet.setSourceMACAddress(config.gatewayExternalInterfaceMac())
.setDestinationMACAddress(config.physicalRouterMac());
ScalableGatewayService gatewayService = getService(ScalableGatewayService.class);
GatewayNode gatewayNode = gatewayService.getGatewayNode(deviceId);
if (gatewayNode.getGatewayExternalInterfaceNames().size() == 0) {
log.error(EXTERNAL_PORT_NULL, deviceId.toString());
return;
}
treatment.setOutput(gatewayService.getGatewayExternalPorts(deviceId).get(0));
ethernet.resetChecksum();
treatment.setOutput(port.number());
packetService.emit(new DefaultOutboundPacket(deviceId, treatment.build(),
ByteBuffer.wrap(ethernet.serialize())));
......
......@@ -15,6 +15,7 @@
*/
package org.onosproject.openstacknetworking.routing;
import com.google.common.collect.Lists;
import org.onlab.packet.ARP;
import org.onlab.packet.EthType;
import org.onlab.packet.Ethernet;
......@@ -34,9 +35,11 @@ import org.onosproject.net.packet.PacketService;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Optional;
import static com.google.common.base.Preconditions.checkNotNull;
......@@ -51,21 +54,23 @@ public class OpenstackRoutingArpHandler {
private final PacketService packetService;
private final OpenstackInterfaceService openstackService;
private final OpenstackNetworkingConfig config;
private final ScalableGatewayService gatewayService;
private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
private static final String NETWORK_FLOATING_IP = "network:floatingip";
/**
* Default constructor.
*
* @param packetService packet service
* @param packetService packet service
* @param openstackService openstackInterface service
* @param config openstackRoutingConfig
* @param gatewayService
*/
OpenstackRoutingArpHandler(PacketService packetService, OpenstackInterfaceService openstackService,
OpenstackNetworkingConfig config) {
OpenstackNetworkingConfig config, ScalableGatewayService gatewayService) {
this.packetService = packetService;
this.openstackService = checkNotNull(openstackService);
this.config = checkNotNull(config);
this.gatewayService = gatewayService;
}
/**
......@@ -79,10 +84,12 @@ public class OpenstackRoutingArpHandler {
.matchEthType(EthType.EtherType.ARP.ethType().toShort())
.build();
packetService.requestPackets(arpSelector,
PacketPriority.CONTROL,
appId,
Optional.of(DeviceId.deviceId(config.gatewayBridgeId())));
getExternalInfo().forEach(deviceId ->
packetService.requestPackets(arpSelector,
PacketPriority.CONTROL,
appId,
Optional.of(deviceId))
);
}
/**
......@@ -138,4 +145,10 @@ public class OpenstackRoutingArpHandler {
}
return port.macAddress();
}
private List<DeviceId> getExternalInfo() {
List<DeviceId> externalInfoList = Lists.newArrayList();
gatewayService.getGatewayDeviceIds().forEach(externalInfoList::add);
return externalInfoList;
}
}
......
......@@ -56,6 +56,7 @@ import org.onosproject.openstacknetworking.OpenstackPortInfo;
import org.onosproject.openstacknetworking.OpenstackRoutingService;
import org.onosproject.openstacknetworking.OpenstackSubjectFactories;
import org.onosproject.openstacknetworking.OpenstackSwitchingService;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.onosproject.store.serializers.KryoNamespaces;
import org.onosproject.store.service.ConsistentMap;
import org.onosproject.store.service.Serializer;
......@@ -113,6 +114,8 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected StorageService storageService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ScalableGatewayService gatewayService;
private ApplicationId appId;
private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
......@@ -444,8 +447,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
if (context.isHandled()) {
return;
} else if (!context.inPacket().receivedFrom().deviceId().toString()
.equals(config.gatewayBridgeId())) {
} else if (!checkGatewayNode(context.inPacket().receivedFrom().deviceId())) {
return;
}
......@@ -472,16 +474,19 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
}
default:
int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
Optional<Port> port =
getExternalPort(pkt.receivedFrom().deviceId(), config.gatewayExternalInterfaceName());
if (port.isPresent()) {
DeviceId deviceId = pkt.receivedFrom().deviceId();
Port port = null;
port = deviceService.getPort(deviceId,
gatewayService.getGatewayExternalPorts(deviceId).get(0));
if (port != null) {
OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
Ip4Address.valueOf(iPacket.getSourceAddress()));
l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
portNum, openstackPort, port.get(), config));
portNum, openstackPort, port, config));
} else {
log.warn("There`s no external interface");
}
break;
}
} else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
......@@ -511,6 +516,10 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
}
private boolean checkGatewayNode(DeviceId deviceId) {
return gatewayService.getGatewayDeviceIds().contains(deviceId);
}
private void clearPortNumMap() {
tpPortNumMap.entrySet().forEach(e -> {
if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
......@@ -600,21 +609,12 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
return;
}
checkNotNull(config.physicalRouterMac());
checkNotNull(config.gatewayBridgeId());
checkNotNull(config.gatewayExternalInterfaceMac());
checkNotNull(config.gatewayExternalInterfaceName());
log.warn("Configured info: {}, {}, {}, {}", config.physicalRouterMac(), config.gatewayBridgeId(),
config.gatewayExternalInterfaceMac(), config.gatewayExternalInterfaceName());
rulePopulator = new OpenstackRoutingRulePopulator(appId,
openstackService, flowObjectiveService, deviceService, driverService, config);
rulePopulator = new OpenstackRoutingRulePopulator(appId, openstackService, flowObjectiveService,
deviceService, driverService, config, gatewayService);
openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService,
openstackService, config, openstackSwitchingService);
openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService, config);
openstackService, config, openstackSwitchingService, gatewayService);
openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService,
config, gatewayService);
openstackIcmpHandler.requestPacket(appId);
openstackArpHandler.requestPacket(appId);
......
......@@ -15,6 +15,7 @@
*/
package org.onosproject.openstacknetworking.routing;
import com.google.common.collect.Lists;
import org.onlab.packet.Ethernet;
import org.onlab.packet.IPv4;
import org.onlab.packet.Ip4Address;
......@@ -25,6 +26,7 @@ import org.onlab.packet.TCP;
import org.onlab.packet.TpPort;
import org.onlab.packet.UDP;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.GroupId;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Port;
......@@ -55,6 +57,7 @@ import org.onosproject.openstackinterface.OpenstackFloatingIP;
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
import org.onosproject.openstacknetworking.OpenstackPortInfo;
import org.onosproject.openstacknetworking.OpenstackRoutingService;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -77,6 +80,7 @@ public class OpenstackRoutingRulePopulator {
private final DeviceService deviceService;
private final DriverService driverService;
private final OpenstackNetworkingConfig config;
private final ScalableGatewayService gatewayService;
private static final String PORTNAME_PREFIX_TUNNEL = "vxlan";
private static final String PORTNAME = "portName";
......@@ -84,6 +88,7 @@ public class OpenstackRoutingRulePopulator {
private static final String PORTNOTNULL = "Port can not be null";
private static final String DEVICENOTNULL = "Device can not be null";
private static final String EXTPORTNOTNULL = "External port can not be null";
private static final String TUNNEL_DESTINATION = "tunnelDst";
private static final int ROUTING_RULE_PRIORITY = 25000;
private static final int FLOATING_RULE_PRIORITY = 42000;
......@@ -107,16 +112,19 @@ public class OpenstackRoutingRulePopulator {
* @param deviceService DeviceService
* @param driverService DriverService
* @param config Configuration for openstack environment
* @param gatewayService scalable gateway service
*/
public OpenstackRoutingRulePopulator(ApplicationId appId, OpenstackInterfaceService openstackService,
FlowObjectiveService flowObjectiveService, DeviceService deviceService,
DriverService driverService, OpenstackNetworkingConfig config) {
DriverService driverService, OpenstackNetworkingConfig config,
ScalableGatewayService gatewayService) {
this.appId = appId;
this.flowObjectiveService = flowObjectiveService;
this.openstackService = checkNotNull(openstackService);
this.deviceService = deviceService;
this.driverService = driverService;
this.config = config;
this.gatewayService = gatewayService;
}
/**
......@@ -155,9 +163,6 @@ public class OpenstackRoutingRulePopulator {
.matchIPDst(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
tBuilder.setEthSrc(externalInterface)
.setEthDst(externalRouter)
.setIpSrc(externalIp);
switch (iPacket.getProtocol()) {
case IPv4.PROTOCOL_TCP:
......@@ -177,19 +182,20 @@ public class OpenstackRoutingRulePopulator {
break;
}
Port port = checkNotNull(getPortOfExternalInterface(), PORTNOTNULL);
tBuilder.setOutput(port.number());
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
.withTreatment(tBuilder.build())
.withFlag(ForwardingObjective.Flag.VERSATILE)
.withPriority(PNAT_RULE_PRIORITY)
.makeTemporary(PNAT_TIMEOUT)
.fromApp(appId)
.add();
flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo);
tBuilder.setIpSrc(externalIp);
gatewayService.getGatewayNodes().forEach(node -> {
tBuilder.setOutput(gatewayService.getGatewayExternalPorts(node.getGatewayDeviceId()).get(0));
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
.withTreatment(tBuilder.build())
.withFlag(ForwardingObjective.Flag.VERSATILE)
.withPriority(PNAT_RULE_PRIORITY)
.makeTemporary(PNAT_TIMEOUT)
.fromApp(appId)
.add();
flowObjectiveService.forward(node.getGatewayDeviceId(), fo);
});
}
......@@ -202,7 +208,6 @@ public class OpenstackRoutingRulePopulator {
private void populatePnatIncomingFlowRules(long vni, Ip4Address externalIp) {
IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
DeviceId deviceId = inboundPacket.receivedFrom().deviceId();
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
......@@ -232,19 +237,29 @@ public class OpenstackRoutingRulePopulator {
break;
}
tBuilder.extension(buildNiciraExtenstion(deviceId, getHostIpfromOpenstackPort(openstackPort)), deviceId)
.setOutput(getTunnelPort(deviceId));
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
.withTreatment(tBuilder.build())
.withFlag(ForwardingObjective.Flag.VERSATILE)
.withPriority(PNAT_RULE_PRIORITY)
.makeTemporary(PNAT_TIMEOUT)
.fromApp(appId)
.add();
getGatewayNodeList().forEach(node -> {
DeviceId deviceId = node.id();
tBuilder.extension(buildNiciraExtenstion(deviceId, getHostIpfromOpenstackPort(openstackPort)), deviceId)
.setOutput(getTunnelPort(deviceId));
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
.withTreatment(tBuilder.build())
.withFlag(ForwardingObjective.Flag.VERSATILE)
.withPriority(PNAT_RULE_PRIORITY)
.makeTemporary(PNAT_TIMEOUT)
.fromApp(appId)
.add();
flowObjectiveService.forward(deviceId, fo);
});
}
flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo);
private List<Device> getGatewayNodeList() {
List<Device> devices = Lists.newArrayList();
gatewayService.getGatewayDeviceIds().forEach(deviceId ->
devices.add(checkNotNull(deviceService.getDevice(deviceId))));
return devices;
}
private Ip4Address getHostIpfromOpenstackPort(OpenstackPort openstackPort) {
......@@ -255,15 +270,15 @@ public class OpenstackRoutingRulePopulator {
private Device getDevicefromOpenstackPort(OpenstackPort openstackPort) {
String openstackPortName = PORTNAME_PREFIX_VM + openstackPort.id().substring(0, 11);
Device device = StreamSupport.stream(deviceService.getDevices().spliterator(), false)
.filter(d -> findPortinDevice(d, openstackPortName))
.filter(d -> findPortinDevice(d.id(), openstackPortName))
.iterator()
.next();
checkNotNull(device, DEVICENOTNULL);
return device;
}
private boolean findPortinDevice(Device d, String openstackPortName) {
Port port = deviceService.getPorts(d.id())
private boolean findPortinDevice(DeviceId deviceId, String openstackPortName) {
Port port = deviceService.getPorts(deviceId)
.stream()
.filter(p -> p.isEnabled() && p.annotations().value(PORTNAME).equals(openstackPortName))
.findAny()
......@@ -274,13 +289,13 @@ public class OpenstackRoutingRulePopulator {
/**
* Builds Nicira extension for tagging remoteIp of vxlan.
*
* @param id Device Id of vxlan source device
* @param deviceId Device Id of vxlan source device
* @param hostIp Remote Ip of vxlan destination device
* @return NiciraExtension Treatment
*/
public ExtensionTreatment buildNiciraExtenstion(DeviceId id, Ip4Address hostIp) {
Driver driver = driverService.getDriver(id);
DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, id));
public ExtensionTreatment buildNiciraExtenstion(DeviceId deviceId, Ip4Address hostIp) {
Driver driver = driverService.getDriver(deviceId);
DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, deviceId));
ExtensionTreatmentResolver resolver = driverHandler.behaviour(ExtensionTreatmentResolver.class);
ExtensionTreatment extensionInstruction =
......@@ -329,7 +344,6 @@ public class OpenstackRoutingRulePopulator {
}
private void populateRuleGatewaytoController(long vni) {
Device gatewayDevice = getGatewayNode();
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
......@@ -346,26 +360,24 @@ public class OpenstackRoutingRulePopulator {
.fromApp(appId)
.add();
flowObjectiveService.forward(gatewayDevice.id(), fo);
getGatewayNodeList().forEach(device -> flowObjectiveService.forward(device.id(), fo));
}
private void populateComputeNodeRules(long vni) {
Device gatewayDevice = getGatewayNode();
StreamSupport.stream(deviceService.getDevices().spliterator(), false)
.filter(d -> !checkGatewayNode(d.id()))
.forEach(d -> populateRuleToGateway(d, gatewayDevice, vni));
.forEach(d -> populateRuleToGatewayBySgw(d.id(),
gatewayService.getGroupIdForGatewayLoadBalance(d.id()), vni));
}
private void populateRuleToGateway(Device d, Device gatewayDevice, long vni) {
private void populateRuleToGatewayBySgw(DeviceId deviceId, GroupId groupId, long vni) {
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchTunnelId(vni)
.matchEthDst(GATEWAYMAC);
tBuilder.extension(buildNiciraExtenstion(d.id(), config.nodes().get(gatewayDevice.id())), d.id())
.setOutput(getTunnelPort(d.id()));
tBuilder.group(groupId);
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
......@@ -375,7 +387,28 @@ public class OpenstackRoutingRulePopulator {
.fromApp(appId)
.add();
flowObjectiveService.forward(d.id(), fo);
flowObjectiveService.forward(deviceId, fo);
}
private void populateRuleToGateway(DeviceId deviceId, Device gatewayDevice, long vni) {
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchTunnelId(vni)
.matchEthDst(GATEWAYMAC);
tBuilder.extension(buildNiciraExtenstion(deviceId, config.nodes().get(gatewayDevice.id())), deviceId)
.setOutput(getTunnelPort(deviceId));
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
.withTreatment(tBuilder.build())
.withFlag(ForwardingObjective.Flag.SPECIFIC)
.withPriority(ROUTING_RULE_PRIORITY)
.fromApp(appId)
.add();
flowObjectiveService.forward(deviceId, fo);
}
private Device getGatewayNode() {
......@@ -383,7 +416,7 @@ public class OpenstackRoutingRulePopulator {
}
private boolean checkGatewayNode(DeviceId deviceId) {
return deviceId.toString().equals(config.gatewayBridgeId());
return gatewayService.getGatewayDeviceIds().stream().anyMatch(dId -> dId.equals(deviceId));
}
private long getVni(String netId) {
......@@ -412,7 +445,7 @@ public class OpenstackRoutingRulePopulator {
}
private void removeRule(DeviceId id, TrafficSelector.Builder sBuilder,
private void removeRule(DeviceId deviceId, TrafficSelector.Builder sBuilder,
ForwardingObjective.Flag flag, int priority) {
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
......@@ -424,7 +457,7 @@ public class OpenstackRoutingRulePopulator {
.fromApp(appId)
.remove();
flowObjectiveService.forward(id, fo);
flowObjectiveService.forward(deviceId, fo);
}
/**
......@@ -442,8 +475,6 @@ public class OpenstackRoutingRulePopulator {
private void populateFloatingIpIncomingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
DeviceId portDeviceId = getDevicefromOpenstackPort(port).id();
Device gatewayNode = getGatewayNode();
Device portNode = deviceService.getDevice(portDeviceId);
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
......@@ -451,29 +482,28 @@ public class OpenstackRoutingRulePopulator {
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
tBuilder.setEthSrc(GATEWAYMAC)
.setEthDst(port.macAddress())
.setIpDst(floatingIP.fixedIpAddress())
.setTunnelId(getVni(port.networkId()))
.extension(buildNiciraExtenstion(gatewayNode.id(),
config.nodes().get(portNode.id())), gatewayNode.id())
.setOutput(getTunnelPort(gatewayNode.id()));
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
.withTreatment(tBuilder.build())
.withFlag(ForwardingObjective.Flag.VERSATILE)
.withPriority(FLOATING_RULE_PRIORITY)
.fromApp(appId)
.add();
flowObjectiveService.forward(getGatewayNode().id(), fo);
getGatewayNodeList().forEach(device -> {
DeviceId deviceId = device.id();
tBuilder.setEthSrc(GATEWAYMAC)
.setEthDst(port.macAddress())
.setIpDst(floatingIP.fixedIpAddress())
.setTunnelId(getVni(port.networkId()))
.extension(buildNiciraExtenstion(deviceId, config.nodes().get(portDeviceId)), deviceId)
.setOutput(getTunnelPort(deviceId));
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
.withTreatment(tBuilder.build())
.withFlag(ForwardingObjective.Flag.VERSATILE)
.withPriority(FLOATING_RULE_PRIORITY)
.fromApp(appId)
.add();
flowObjectiveService.forward(deviceId, fo);
});
}
private void populateFloatingIpOutgoingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
Port outputPort = checkNotNull(getPortOfExternalInterface(), PORTNOTNULL);
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
......@@ -481,20 +511,25 @@ public class OpenstackRoutingRulePopulator {
.matchTunnelId(getVni(port.networkId()))
.matchIPSrc(IpPrefix.valueOf(floatingIP.fixedIpAddress(), 32));
tBuilder.setIpSrc(floatingIP.floatingIpAddress())
.setEthSrc(MacAddress.valueOf(config.gatewayExternalInterfaceMac()))
.setEthDst(MacAddress.valueOf(config.physicalRouterMac()))
.setOutput(outputPort.number());
getGatewayNodeList().forEach(device -> {
DeviceId deviceId = device.id();
tBuilder.setIpSrc(floatingIP.floatingIpAddress())
.setOutput(getExternalPortNum(deviceId));
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
.withTreatment(tBuilder.build())
.withFlag(ForwardingObjective.Flag.VERSATILE)
.withPriority(FLOATING_RULE_PRIORITY)
.fromApp(appId)
.add();
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
.withTreatment(tBuilder.build())
.withFlag(ForwardingObjective.Flag.VERSATILE)
.withPriority(FLOATING_RULE_PRIORITY)
.fromApp(appId)
.add();
flowObjectiveService.forward(getGatewayNode().id(), fo);
flowObjectiveService.forward(deviceId, fo);
});
}
private PortNumber getExternalPortNum(DeviceId deviceId) {
return checkNotNull(gatewayService.getGatewayExternalPorts(deviceId).get(0), PORTNOTNULL);
}
/**
......@@ -514,38 +549,39 @@ public class OpenstackRoutingRulePopulator {
sIncomingBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
removeRule(getGatewayNode().id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
removeRule(getGatewayNode().id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
getGatewayNodeList().forEach(device -> {
removeRule(device.id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
removeRule(device.id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
});
}
/**
* Populates L3 rules for east to west traffic.
*
* @param p target VM
* @param openstackPort target VM
* @param targetList target openstackRouterInterfaces
*/
public void populateL3Rules(OpenstackPort p, List<OpenstackRouterInterface> targetList) {
Device device = getDevicefromOpenstackPort(p);
Port port = getPortFromOpenstackPort(device, p);
Ip4Address vmIp = (Ip4Address) p.fixedIps().values().iterator().next();
public void populateL3Rules(OpenstackPort openstackPort, List<OpenstackRouterInterface> targetList) {
Device device = getDevicefromOpenstackPort(openstackPort);
Port port = getPortFromOpenstackPort(device, openstackPort);
Ip4Address vmIp = openstackPort.fixedIps().values().iterator().next();
if (port == null) {
return;
}
targetList.forEach(routerInterface -> {
OpenstackPort openstackPort = openstackService.port(routerInterface.portId());
long vni = getVni(openstackPort.networkId());
long vni = getVni(openstackService.port(routerInterface.portId()).networkId());
if (vmIp == null) {
return;
}
populateL3RulestoSameNode(vmIp, p, port, device, vni);
populateL3RulestoSameNode(vmIp, openstackPort, port, device, vni);
deviceService.getAvailableDevices().forEach(d -> {
if (!d.equals(device) && !d.equals(getGatewayNode())) {
populateL3RulestoDifferentNode(vmIp, vni, d.id(), getHostIpfromOpenstackPort(p));
if (!d.equals(device) && !checkGatewayNode(d.id())) {
populateL3RulestoDifferentNode(vmIp, vni, d.id(), getHostIpfromOpenstackPort(openstackPort));
}
});
......@@ -618,7 +654,7 @@ public class OpenstackRoutingRulePopulator {
OpenstackRoutingService routingService = getService(OpenstackRoutingService.class);
deviceService.getAvailableDevices().forEach(d -> {
if (!d.equals(getGatewayNode())) {
if (!checkGatewayNode(d.id())) {
routerInterfaces.forEach(routerInterface -> {
String networkId = routingService.networkIdForRouterInterface(routerInterface.portId());
long vni = getVni(networkId);
......
{
"apps" : {
"org.onosproject.scalablegateway" : {
"scalablegateway" : {
"nodes" : [
{
"bridgeId" : "of:0000000000000003",
"dataPlaneIp" : "10.62.148.17",
"gatewayExternalInterfaceName" : "veth0"
}]
}
}
}
}
......@@ -66,6 +66,15 @@ public final class GatewayNode {
}
/**
* Returns GatewayNode builder object.
*
* @return GatewayNode builder
*/
public static GatewayNode.Builder builder() {
return new Builder();
}
/**
* GatewayNode Builder class.
*/
public static final class Builder {
......
......@@ -17,6 +17,7 @@ package org.onosproject.scalablegateway.api;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.onlab.packet.Ip4Address;
......@@ -28,9 +29,10 @@ import org.slf4j.Logger;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.StreamSupport;
import static org.slf4j.LoggerFactory.getLogger;
import static org.onosproject.net.config.Config.FieldPresence.MANDATORY;
import static org.slf4j.LoggerFactory.getLogger;
/**
* Configuration object for OpensatckNode service.
......@@ -79,9 +81,21 @@ public class GatewayNodeConfig extends Config<ApplicationId> {
@Override
public boolean isValid() {
return hasOnlyFields(NODES, BRIDGE_ID, DATAPLANE_IP, EXTERNAL_INTERFACE_NAME) &&
isIpAddress(DATAPLANE_IP, MANDATORY) && isString(BRIDGE_ID, MANDATORY) &&
isString(EXTERNAL_INTERFACE_NAME, MANDATORY);
JsonNode jsonNodes = object.get(NODES);
if (jsonNodes == null) {
return false;
}
return hasOnlyFields(NODES)
&& StreamSupport.stream(jsonNodes.spliterator(), false).allMatch(this::checkValid);
}
private boolean checkValid(JsonNode jsonNode) {
ObjectNode objectNode = (ObjectNode) jsonNode;
return isString(objectNode, BRIDGE_ID, MANDATORY)
&& isIpAddress(objectNode, DATAPLANE_IP, MANDATORY)
&& isString(objectNode, EXTERNAL_INTERFACE_NAME, MANDATORY);
}
}
......