홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Kyuhwi Choi
Committed by Gerrit Code Review
5e65e2cc 1 parent 9cc05f0b

[ONOS-4205] support east-west routing in openstackRouting 

  - Supports east-west routing traffic
  - Fixes javadocs

Change-Id: I23d9b9497cc2be667fbc9554812c7f5b49c35364
Inline Side-by-side
Showing 3 changed files with 292 additions and 55 deletions
......@@ -20,71 +20,78 @@ import org.onosproject.openstackinterface.OpenstackRouter;
import org.onosproject.openstackinterface.OpenstackRouterInterface;
/**
* The Interface of Openstack Routing.
* Supports L3 management REST API for openstack.
*/
public interface OpenstackRoutingService {
/**
* Stores the Floating IP information created by Openstack.
* Stores the floating IP information created by openstack.
*
* @param openstackFloatingIP Floating IP information
* @param openstackFloatingIp Floating IP information
*/
void createFloatingIP(OpenstackFloatingIP openstackFloatingIP);
void createFloatingIP(OpenstackFloatingIP openstackFloatingIp);
/**
* Updates flow rules corresponding to the Floating IP information updated by Openstack.
* Updates flow rules corresponding to the floating IP information updated by openstack.
*
* @param openstackFloatingIP Floating IP information
* @param openstackFloatingIp Floating IP information
*/
void updateFloatingIP(OpenstackFloatingIP openstackFloatingIP);
void updateFloatingIP(OpenstackFloatingIP openstackFloatingIp);
/**
* Removes flow rules corresponding to Floating IP information removed by Openstack.
* Removes flow rules corresponding to floating IP information removed by openstack.
*
* @param id Deleted Floating IP`s ID
*/
void deleteFloatingIP(String id);
/**
* Stores the router information created by Openstack.
* Stores the router information created by openstack.
*
* @param openstackRouter Floating IP information
* @param openstackRouter Router information
*/
void createRouter(OpenstackRouter openstackRouter);
/**
* Updates flow rules corresponding to the router information updated by Openstack.
* Updates flow rules corresponding to the router information updated by openstack.
*
* @param openstackRouter Router information
*/
void updateRouter(OpenstackRouter openstackRouter);
/**
* Removes flow rules corresponding to the router information removed by Openstack.
* Removes flow rules corresponding to the router information removed by openstack.
*
* @param id Deleted router`s ID
*/
void deleteRouter(String id);
/**
* Updates flow rules corresponding to the router information updated by Openstack.
* Updates flow rules corresponding to the router information updated by openstack.
*
* @param openstackRouterInterface Router information
* @param openstackRouterInterface Router interface information
*/
void updateRouterInterface(OpenstackRouterInterface openstackRouterInterface);
/**
* Removes flow rules corresponding to the router information removed by Openstack.
* Removes flow rules corresponding to the router information removed by openstack.
*
* @param openstackRouterInterface Router information
* @param openstackRouterInterface Router interface information
*/
void removeRouterInterface(OpenstackRouterInterface openstackRouterInterface);
/**
* Checks floatingIp deassociation when corresponding deleted vm.
* Checks floatingIp disassociation when corresponding deleted vm.
*
* @param portId Deleted vm
* @param portInfo stored information about deleted vm
*/
void checkDisassociatedFloatingIp(String portId, OpenstackPortInfo portInfo);
/**
* Returns network id for routerInterface.
*
* @param portId routerInterface`s port id
*/
String networkIdForRouterInterface(String portId);
}
......
......@@ -37,6 +37,8 @@ import org.onosproject.net.config.NetworkConfigEvent;
import org.onosproject.net.config.NetworkConfigListener;
import org.onosproject.net.config.NetworkConfigRegistry;
import org.onosproject.net.config.NetworkConfigService;
import org.onosproject.net.device.DeviceEvent;
import org.onosproject.net.device.DeviceListener;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.driver.DriverService;
import org.onosproject.net.flowobjective.FlowObjectiveService;
......@@ -115,12 +117,15 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
private ApplicationId appId;
private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
private ConsistentMap<String, OpenstackFloatingIP> floatingIpMap; // Map<FloatingIp`s Id, FloatingIp object>
// Map<RouterInterface`s portId, Corresponded port`s network id>
private ConsistentMap<String, String> routerInterfaceMap;
private static final String APP_ID = "org.onosproject.openstackrouting";
private static final String PORT_NAME = "portName";
private static final String PORTNAME_PREFIX_VM = "tap";
private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
private static final String FLOATING_IP_MAP_NAME = "openstackrouting-floatingip";
private static final String TP_PORT_MAP_NAME = "openstackrouting-portnum";
private static final String TP_PORT_MAP_NAME = "openstackrouting-tpportnum";
private static final String ROUTER_INTERFACE_MAP_NAME = "openstackrouting-routerinterface";
private static final String COLON = ":";
private static final int PNAT_PORT_EXPIRE_TIME = 1200 * 1000;
private static final int TP_PORT_MINIMUM_NUM = 1024;
......@@ -152,7 +157,13 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
.register(Integer.class)
.register(String.class);
private static final KryoNamespace.Builder ROUTER_INTERFACE_SERIALIZER = KryoNamespace.newBuilder()
.register(KryoNamespaces.API)
.register(KryoNamespaces.MISC)
.register(String.class);
private InternalPacketProcessor internalPacketProcessor = new InternalPacketProcessor();
private InternalDeviceListener internalDeviceListener = new InternalDeviceListener();
private ExecutorService l3EventExecutorService =
Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "L3-event"));
private ExecutorService icmpEventExecutorService =
......@@ -170,6 +181,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
packetService.addProcessor(internalPacketProcessor, PacketProcessor.director(1));
configRegistry.registerConfigFactory(configFactory);
configService.addListener(configListener);
deviceService.addListener(internalDeviceListener);
floatingIpMap = storageService.<String, OpenstackFloatingIP>consistentMapBuilder()
.withSerializer(Serializer.using(FLOATING_IP_SERIALIZER.build()))
......@@ -181,52 +193,59 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
.withName(TP_PORT_MAP_NAME)
.withApplicationId(appId)
.build();
routerInterfaceMap = storageService.<String, String>consistentMapBuilder()
.withSerializer(Serializer.using(ROUTER_INTERFACE_SERIALIZER.build()))
.withName(ROUTER_INTERFACE_MAP_NAME)
.withApplicationId(appId)
.build();
readConfiguration();
log.info("onos-openstackrouting started");
log.info("started");
}
@Deactivate
protected void deactivate() {
packetService.removeProcessor(internalPacketProcessor);
deviceService.removeListener(internalDeviceListener);
l3EventExecutorService.shutdown();
icmpEventExecutorService.shutdown();
arpEventExecutorService.shutdown();
floatingIpMap.clear();
tpPortNumMap.clear();
routerInterfaceMap.clear();
log.info("onos-openstackrouting stopped");
log.info("stopped");
}
@Override
public void createFloatingIP(OpenstackFloatingIP openstackFloatingIP) {
floatingIpMap.put(openstackFloatingIP.id(), openstackFloatingIP);
public void createFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
}
@Override
public void updateFloatingIP(OpenstackFloatingIP openstackFloatingIP) {
if (!floatingIpMap.containsKey(openstackFloatingIP.id())) {
log.warn("There`s no information about {} in FloatingIpMap", openstackFloatingIP.id());
public void updateFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
if (!floatingIpMap.containsKey(openstackFloatingIp.id())) {
log.warn("There`s no information about {} in FloatingIpMap", openstackFloatingIp.id());
return;
}
if (openstackFloatingIP.portId() == null || openstackFloatingIP.portId().equals("null")) {
OpenstackFloatingIP floatingIP = floatingIpMap.get(openstackFloatingIP.id()).value();
if (openstackFloatingIp.portId() == null || openstackFloatingIp.portId().equals("null")) {
OpenstackFloatingIP floatingIp = floatingIpMap.get(openstackFloatingIp.id()).value();
OpenstackPortInfo portInfo = openstackSwitchingService.openstackPortInfo()
.get(PORTNAME_PREFIX_VM.concat(floatingIP.portId().substring(0, 11)));
.get(PORTNAME_PREFIX_VM.concat(floatingIp.portId().substring(0, 11)));
if (portInfo == null) {
log.warn("There`s no portInfo information about portId {}", floatingIP.portId());
log.warn("There`s no portInfo information about portId {}", floatingIp.portId());
return;
}
l3EventExecutorService.execute(
new OpenstackFloatingIPHandler(rulePopulator, floatingIP, false, portInfo));
floatingIpMap.replace(floatingIP.id(), openstackFloatingIP);
new OpenstackFloatingIPHandler(rulePopulator, floatingIp, false, portInfo));
floatingIpMap.replace(floatingIp.id(), openstackFloatingIp);
} else {
floatingIpMap.put(openstackFloatingIP.id(), openstackFloatingIP);
floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
l3EventExecutorService.execute(
new OpenstackFloatingIPHandler(rulePopulator, openstackFloatingIP, true, null));
new OpenstackFloatingIPHandler(rulePopulator, openstackFloatingIp, true, null));
}
}
......@@ -280,11 +299,69 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
List<OpenstackRouterInterface> routerInterfaces = Lists.newArrayList();
routerInterfaces.add(routerInterface);
checkExternalConnection(getOpenstackRouter(routerInterface.id()), routerInterfaces);
setL3Connection(getOpenstackRouter(routerInterface.id()), null);
routerInterfaceMap.put(routerInterface.portId(), openstackService.port(routerInterface.portId()).networkId());
}
private void setL3Connection(OpenstackRouter openstackRouter, OpenstackPort openstackPort) {
Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(openstackRouter);
if (interfaceList.size() < 2) {
return;
}
if (openstackPort == null) {
interfaceList.forEach(i -> {
OpenstackPort interfacePort = openstackService.port(i.portId());
openstackService.ports()
.stream()
.filter(p -> p.networkId().equals(interfacePort.networkId())
&& !p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
.forEach(p -> rulePopulator.populateL3Rules(p,
getL3ConnectionList(p.networkId(), interfaceList)));
});
} else {
rulePopulator.populateL3Rules(openstackPort, getL3ConnectionList(openstackPort.networkId(), interfaceList));
}
}
private List<OpenstackRouterInterface> getL3ConnectionList(String networkId,
Collection<OpenstackRouterInterface> interfaceList) {
List<OpenstackRouterInterface> targetList = Lists.newArrayList();
interfaceList.forEach(i -> {
OpenstackPort port = openstackService.port(i.portId());
if (!port.networkId().equals(networkId)) {
targetList.add(i);
}
});
return targetList;
}
@Override
public void removeRouterInterface(OpenstackRouterInterface routerInterface) {
OpenstackRouter router = openstackService.router(routerInterface.id());
Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(router);
if (interfaceList.size() == 1) {
List<OpenstackRouterInterface> newList = Lists.newArrayList();
newList.add(routerInterface);
interfaceList.forEach(i -> removeL3RulesForRouterInterface(i, router, newList));
}
removeL3RulesForRouterInterface(routerInterface, router, null);
rulePopulator.removeExternalRules(routerInterface);
routerInterfaceMap.remove(routerInterface.portId());
}
private void removeL3RulesForRouterInterface(OpenstackRouterInterface routerInterface, OpenstackRouter router,
List<OpenstackRouterInterface> newList) {
openstackService.ports(routerInterfaceMap.get(routerInterface.portId()).value()).forEach(p -> {
Ip4Address vmIp = (Ip4Address) p.fixedIps().values().toArray()[0];
if (newList == null) {
rulePopulator.removeL3Rules(vmIp,
getL3ConnectionList(p.networkId(), getOpenstackRouterInterface(router)));
} else {
rulePopulator.removeL3Rules(vmIp, newList);
}
}
);
}
@Override
......@@ -313,6 +390,11 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
}
}
@Override
public String networkIdForRouterInterface(String portId) {
return routerInterfaceMap.get(portId).value();
}
private Collection<OpenstackFloatingIP> associatedFloatingIps() {
List<OpenstackFloatingIP> fIps = Lists.newArrayList();
floatingIpMap.values()
......@@ -327,10 +409,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
openstackService.ports()
.stream()
.filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
.forEach(p -> {
OpenstackRouterInterface routerInterface = portToRouterInterface(p);
updateRouterInterface(routerInterface);
})
.forEach(p -> updateRouterInterface(portToRouterInterface(p)))
);
}
......@@ -382,14 +461,13 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
Optional<Port> port =
getExternalPort(pkt.receivedFrom().deviceId(), config.gatewayExternalInterfaceName());
if (!port.isPresent()) {
log.warn("There`s no external interface");
} else {
if (port.isPresent()) {
OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
Ip4Address.valueOf(iPacket.getSourceAddress()));
l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
portNum, openstackPort, port.get(), config));
} else {
log.warn("There`s no external interface");
}
break;
}
......@@ -471,22 +549,20 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
List<OpenstackRouterInterface> interfaces = Lists.newArrayList();
openstackService.ports()
.stream()
.filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
.filter(p -> p.deviceId().equals(router.id()))
.forEach(p -> {
interfaces.add(portToRouterInterface(p));
});
.filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
&& p.deviceId().equals(router.id()))
.forEach(p -> interfaces.add(portToRouterInterface(p)));
return interfaces;
}
private OpenstackRouter getOpenstackRouter(String id) {
return openstackService.routers().stream().filter(r ->
r.id().equals(id)).findAny().orElse(null);
r.id().equals(id)).iterator().next();
}
private OpenstackPort getOpenstackPort(MacAddress sourceMac, Ip4Address ip4Address) {
OpenstackPort openstackPort = openstackService.ports().stream()
.filter(p -> p.macAddress().equals(sourceMac)).findFirst().orElse(null);
.filter(p -> p.macAddress().equals(sourceMac)).iterator().next();
return openstackPort.fixedIps().values().stream().filter(ip ->
ip.equals(ip4Address)).count() > 0 ? openstackPort : null;
}
......@@ -534,4 +610,38 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
}
}
}
private class InternalDeviceListener implements DeviceListener {
@Override
public void event(DeviceEvent deviceEvent) {
if (deviceEvent.type() == DeviceEvent.Type.PORT_UPDATED) {
Port port = deviceEvent.port();
OpenstackPortInfo portInfo = openstackSwitchingService.openstackPortInfo()
.get(port.annotations().value(PORT_NAME));
OpenstackPort openstackPort = openstackService.port(port);
OpenstackPort interfacePort = openstackService.ports()
.stream()
.filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
&& p.networkId().equals(openstackPort.networkId()))
.findAny()
.orElse(null);
if (portInfo == null && openstackPort == null) {
log.warn("As delete event timing issue between routing and switching, Can`t delete L3 rules");
return;
}
if ((port.isEnabled()) && (interfacePort != null)) {
OpenstackRouterInterface routerInterface = portToRouterInterface(interfacePort);
l3EventExecutorService.execute(() ->
setL3Connection(getOpenstackRouter(routerInterface.id()), openstackPort));
} else if (interfacePort != null) {
OpenstackRouterInterface routerInterface = portToRouterInterface(interfacePort);
l3EventExecutorService.execute(() -> rulePopulator.removeL3Rules(portInfo.ip(),
getL3ConnectionList(portInfo.networkId(),
getOpenstackRouterInterface(getOpenstackRouter(routerInterface.id())))));
}
}
}
}
}
......
......@@ -54,12 +54,15 @@ import org.onosproject.openstackinterface.OpenstackSubnet;
import org.onosproject.openstackinterface.OpenstackFloatingIP;
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
import org.onosproject.openstacknetworking.OpenstackPortInfo;
import org.onosproject.openstacknetworking.OpenstackRoutingService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.stream.StreamSupport;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.onlab.osgi.DefaultServiceDirectory.getService;
/**
* Populates Routing Flow Rules.
......@@ -99,11 +102,11 @@ public class OpenstackRoutingRulePopulator {
* The constructor of openstackRoutingRulePopulator.
*
* @param appId Caller`s appId
* @param openstackService OpenstackNetworkingService
* @param openstackService Opestack REST request handler
* @param flowObjectiveService FlowObjectiveService
* @param deviceService DeviceService
* @param driverService DriverService
* @param config OpenstackRoutingConfig
* @param config Configuration for openstack environment
*/
public OpenstackRoutingRulePopulator(ApplicationId appId, OpenstackInterfaceService openstackService,
FlowObjectiveService flowObjectiveService, DeviceService deviceService,
......@@ -187,6 +190,7 @@ public class OpenstackRoutingRulePopulator {
.add();
flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo);
}
private Port getPortOfExternalInterface() {
......@@ -252,8 +256,8 @@ public class OpenstackRoutingRulePopulator {
String openstackPortName = PORTNAME_PREFIX_VM + openstackPort.id().substring(0, 11);
Device device = StreamSupport.stream(deviceService.getDevices().spliterator(), false)
.filter(d -> findPortinDevice(d, openstackPortName))
.findAny()
.orElse(null);
.iterator()
.next();
checkNotNull(device, DEVICENOTNULL);
return device;
}
......@@ -268,7 +272,7 @@ public class OpenstackRoutingRulePopulator {
}
/**
* Builds NiciraExtension for tagging remoteIp of vxlan.
* Builds Nicira extension for tagging remoteIp of vxlan.
*
* @param id Device Id of vxlan source device
* @param hostIp Remote Ip of vxlan destination device
......@@ -401,8 +405,8 @@ public class OpenstackRoutingRulePopulator {
StreamSupport.stream(deviceService.getDevices().spliterator(), false)
.forEach(d -> {
ForwardingObjective.Flag flag = checkGatewayNode(d.id()) ?
ForwardingObjective.Flag.VERSATILE : ForwardingObjective.Flag.SPECIFIC;
ForwardingObjective.Flag.VERSATILE :
ForwardingObjective.Flag.SPECIFIC;
removeRule(d.id(), sBuilder, flag, ROUTING_RULE_PRIORITY);
});
......@@ -514,4 +518,120 @@ public class OpenstackRoutingRulePopulator {
removeRule(getGatewayNode().id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
}
/**
* Populates L3 rules for east to west traffic.
*
* @param p target VM
* @param targetList target openstackRouterInterfaces
*/
public void populateL3Rules(OpenstackPort p, List<OpenstackRouterInterface> targetList) {
Device device = getDevicefromOpenstackPort(p);
Port port = getPortFromOpenstackPort(device, p);
Ip4Address vmIp = (Ip4Address) p.fixedIps().values().iterator().next();
if (port == null) {
return;
}
targetList.forEach(routerInterface -> {
OpenstackPort openstackPort = openstackService.port(routerInterface.portId());
long vni = getVni(openstackPort.networkId());
if (vmIp == null) {
return;
}
populateL3RulestoSameNode(vmIp, p, port, device, vni);
deviceService.getAvailableDevices().forEach(d -> {
if (!d.equals(device) && !d.equals(getGatewayNode())) {
populateL3RulestoDifferentNode(vmIp, vni, d.id(), getHostIpfromOpenstackPort(p));
}
});
});
}
private void populateL3RulestoDifferentNode(Ip4Address vmIp, long vni, DeviceId deviceId, Ip4Address hostIp) {
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchTunnelId(vni)
.matchIPDst(vmIp.toIpPrefix());
tBuilder.extension(buildNiciraExtenstion(deviceId, hostIp), deviceId)
.setOutput(getTunnelPort(deviceId));
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
.withTreatment(tBuilder.build())
.withPriority(ROUTING_RULE_PRIORITY)
.withFlag(ForwardingObjective.Flag.SPECIFIC)
.fromApp(appId)
.add();
flowObjectiveService.forward(deviceId, fo);
}
private void populateL3RulestoSameNode(Ip4Address vmIp, OpenstackPort p, Port port, Device device, long vni) {
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchIPDst(vmIp.toIpPrefix())
.matchTunnelId(vni);
tBuilder.setEthDst(p.macAddress())
.setOutput(port.number());
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
.withTreatment(tBuilder.build())
.withPriority(ROUTING_RULE_PRIORITY)
.withFlag(ForwardingObjective.Flag.SPECIFIC)
.fromApp(appId)
.add();
flowObjectiveService.forward(device.id(), fo);
}
private Port getPortFromOpenstackPort(Device device, OpenstackPort p) {
String openstackPortName = PORTNAME_PREFIX_VM + p.id().substring(0, 11);
return deviceService.getPorts(device.id())
.stream()
.filter(pt -> pt.annotations().value(PORTNAME).equals(openstackPortName))
.findAny()
.orElse(null);
}
/**
* Removes L3 rules for routerInterface events.
*
* @param vmIp Corresponding Vm ip
* @param routerInterfaces Corresponding routerInterfaces
*/
public void removeL3Rules(Ip4Address vmIp, List<OpenstackRouterInterface> routerInterfaces) {
if (vmIp == null) {
return;
}
OpenstackRoutingService routingService = getService(OpenstackRoutingService.class);
deviceService.getAvailableDevices().forEach(d -> {
if (!d.equals(getGatewayNode())) {
routerInterfaces.forEach(routerInterface -> {
String networkId = routingService.networkIdForRouterInterface(routerInterface.portId());
long vni = getVni(networkId);
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchIPDst(vmIp.toIpPrefix())
.matchTunnelId(vni);
removeRule(d.id(), sBuilder, ForwardingObjective.Flag.SPECIFIC, ROUTING_RULE_PRIORITY);
});
}
});
}
}
......