홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Daniel Park
Committed by Gerrit Code Review
2319390b 1 parent 24439fe2

[ONOS-3948] Seperate mgmt and data network in OpenstackSwitching/RoutingService 

- Supports the seperation of management and data network

Change-Id: I178dbe2af241123c5181f94a7b46fc15b4cb37c7
Inline Side-by-side
Showing 10 changed files with 369 additions and 277 deletions
......@@ -15,13 +15,13 @@
*/
package org.onosproject.openstackinterface;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.onlab.packet.Ip4Address;
import org.onlab.packet.IpAddress;
import org.onlab.packet.MacAddress;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import static com.google.common.base.Preconditions.checkNotNull;
......@@ -40,31 +40,31 @@ public final class OpenstackPort {
private PortStatus status;
private String name;
private Map<IpAddress, MacAddress> allowedAddressPairs;
private ImmutableMap<IpAddress, MacAddress> allowedAddressPairs;
private boolean adminStateUp;
private String networkId;
private String tenantId;
private String deviceOwner;
private MacAddress macAddress;
// <subnet id, ip address>
private HashMap<String, Ip4Address> fixedIps;
private ImmutableMap<String, Ip4Address> fixedIps;
private String id;
private Collection<String> securityGroups;
private String deviceId;
private OpenstackPort(PortStatus status, String name, Map<IpAddress, MacAddress> allowedAddressPairs,
boolean adminStateUp, String networkId, String tenantId,
String deviceOwner, MacAddress macAddress, HashMap fixedIps,
String deviceOwner, MacAddress macAddress, Map<String, Ip4Address> fixedIps,
String id, Collection<String> securityGroups, String deviceId) {
this.status = status;
this.name = name;
this.allowedAddressPairs = checkNotNull(allowedAddressPairs);
this.allowedAddressPairs = checkNotNull(ImmutableMap.copyOf(allowedAddressPairs));
this.adminStateUp = adminStateUp;
this.networkId = checkNotNull(networkId);
this.tenantId = checkNotNull(tenantId);
this.deviceOwner = deviceOwner;
this.macAddress = checkNotNull(macAddress);
this.fixedIps = checkNotNull(fixedIps);
this.fixedIps = checkNotNull(ImmutableMap.copyOf(fixedIps));
this.id = checkNotNull(id);
this.securityGroups = securityGroups;
this.deviceId = deviceId;
......@@ -149,7 +149,7 @@ public final class OpenstackPort {
*
* @return fixed IP info
*/
public HashMap fixedIps() {
public Map<String, Ip4Address> fixedIps() {
return fixedIps;
}
......@@ -194,7 +194,7 @@ public final class OpenstackPort {
private String deviceOwner;
private MacAddress macAddress;
// list of hash map <subnet id, ip address>
private HashMap<String, Ip4Address> fixedIps;
private Map<String, Ip4Address> fixedIps;
private String id;
private Collection<String> securityGroups;
private String deviceId;
......@@ -305,7 +305,7 @@ public final class OpenstackPort {
* @param fixedIpList Fixed IP info
* @return Builder object
*/
public Builder fixedIps(HashMap<String, Ip4Address> fixedIpList) {
public Builder fixedIps(Map<String, Ip4Address> fixedIpList) {
fixedIps.putAll(fixedIpList);
return this;
......
......@@ -30,7 +30,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import static com.google.common.base.Preconditions.checkNotNull;
......@@ -64,7 +63,7 @@ public class OpenstackPortCodec extends JsonCodec<OpenstackPort> {
public OpenstackPort decode(ObjectNode json, CodecContext context) {
checkNotNull(json);
HashMap<String, Ip4Address> fixedIpMap = new HashMap<>();
Map<String, Ip4Address> fixedIpMap = Maps.newHashMap();
JsonNode portInfo = json.get(PORT);
if (portInfo == null) {
portInfo = json;
......
{
"apps" : {
"org.onosproject.openstackrouting" : {
"openstackrouting" : {
"physicalRouterMac" : "2a:a1:8a:89:dd:a4",
"gatewayBridgeId" : "of:0000000000000003",
"gatewayExternalInterfaceName" : "veth0",
"gatewayExternalInterfaceMac" : "be:15:c6:b0:df:9f"
}
},
"org.onosproject.openstacknode" : {
"openstacknode" : {
"nodes" : [
{
"hostname" : "compute-01",
"ovsdbIp" : "192.168.56.112",
"ovsdbPort" : "6640",
"bridgeId" : "of:0000000000000001",
"openstackNodeType" : "COMPUTENODE"
},
{
"hostname" : "compute-02",
"ovsdbIp" : "192.168.56.113",
"ovsdbPort" : "6640",
"bridgeId" : "of:0000000000000002",
"openstackNodeType" : "COMPUTENODE"
},
{
"hostname" : "network",
"ovsdbIp" : "192.168.56.114",
"ovsdbPort" : "6640",
"bridgeId" : "of:0000000000000003",
"openstackNodeType" : "GATEWAYNODE",
"gatewayExternalInterfaceName" : "veth0",
"gatewayExternalInterfaceMac" : "be:15:c6:b0:df:9f"
}
]
}
},
"org.onosproject.openstackinterface" : {
"openstackinterface" : {
"neutron_server" : "http://192.168.56.111:9696/v2.0/",
"keystone_server" : "http://192.168.56.111:5000/v2.0/",
"user_name" : "admin",
"password" : "nova"
}
}
"apps" : {
"org.onosproject.openstackrouting" : {
"openstackrouting" : {
"physicalRouterMac" : "36:e3:f7:7b:cc:52",
"gatewayBridgeId" : "of:0000000000000003",
"gatewayExternalInterfaceName" : "veth0",
"gatewayExternalInterfaceMac" : "96:87:bc:64:99:ad",
"nodes" : [
{
"dataPlaneIp" : "192.168.57.103",
"bridgeId" : "of:0000000000000001"
},
{
"dataPlaneIp" : "192.168.57.104",
"bridgeId" : "of:0000000000000002"
},
{
"dataPlaneIp" : "192.168.57.105",
"bridgeId" : "of:0000000000000003"
}
]
}
},
"devices" : {
"of:0000000000000001" : {
"basic" : {
"driver" : "sona"
}
},
"of:0000000000000002" : {
"basic" : {
"driver" : "sona"
}
}
"org.onosproject.openstackswitching" : {
"openstackswitching" : {
"nodes" : [
{
"dataPlaneIp" : "192.168.57.103",
"bridgeId" : "of:0000000000000001"
},
{
"dataPlaneIp" : "192.168.57.104",
"bridgeId" : "of:0000000000000002"
},
{
"dataPlaneIp" : "192.168.57.105",
"bridgeId" : "of:0000000000000003"
}
]
}
},
"org.onosproject.openstacknode" : {
"openstacknode" : {
"nodes" : [
{
"hostname" : "compute-01",
"ovsdbIp" : "192.168.56.121",
"ovsdbPort" : "6640",
"bridgeId" : "of:0000000000000001",
"openstackNodeType" : "COMPUTENODE"
},
{
"hostname" : "compute-02",
"ovsdbIp" : "192.168.56.122",
"ovsdbPort" : "6640",
"bridgeId" : "of:0000000000000002",
"openstackNodeType" : "COMPUTENODE"
},
{
"hostname" : "network",
"ovsdbIp" : "192.168.56.114",
"ovsdbPort" : "6640",
"bridgeId" : "of:0000000000000003",
"openstackNodeType" : "GATEWAYNODE",
"gatewayExternalInterfaceName" : "veth0",
"gatewayExternalInterfaceMac" : "96:87:bc:64:99:ad"
}
]
}
},
"org.onosproject.openstackinterface" : {
"openstackinterface" : {
"neutron_server" : "http://192.168.56.118:9696/v2.0/",
"keystone_server" : "http://192.168.56.118:5000/v2.0/",
"user_name" : "admin",
"password" : "rocks"
}
}
},
"devices" : {
"of:0000000000000001" : {
"basic" : {
"driver" : "sona"
}
},
"of:0000000000000002" : {
"basic" : {
"driver" : "sona"
}
}
}
}
......
......@@ -15,10 +15,16 @@
*/
package org.onosproject.openstacknetworking.routing;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Maps;
import org.onlab.packet.Ip4Address;
import org.onosproject.core.ApplicationId;
import org.onosproject.net.DeviceId;
import org.onosproject.net.config.Config;
import org.slf4j.Logger;
import java.util.Map;
import static org.slf4j.LoggerFactory.getLogger;
/**
......@@ -31,6 +37,35 @@ public class OpenstackRoutingConfig extends Config<ApplicationId> {
public static final String GATEWAY_BRIDGE_ID = "gatewayBridgeId";
public static final String GATEWAY_EXTERNAL_INTERFACE_NAME = "gatewayExternalInterfaceName";
public static final String GATEWAY_EXTERNAL_INTERFACE_MAC = "gatewayExternalInterfaceMac";
public static final String NODES = "nodes";
public static final String DATAPLANE_IP = "dataPlaneIp";
public static final String BRIDGE_ID = "bridgeId";
/**
* Returns the data plane IP map of nodes read from network config.
*
* @return data plane IP map
*/
public Map<DeviceId, Ip4Address> nodes() {
Map<DeviceId, Ip4Address> nodeMap = Maps.newHashMap();
JsonNode jsonNodes = object.get(NODES);
if (jsonNodes == null) {
log.error("There's no node information");
return null;
}
jsonNodes.forEach(jsonNode -> {
try {
nodeMap.putIfAbsent(DeviceId.deviceId(jsonNode.path(BRIDGE_ID).asText()),
Ip4Address.valueOf(jsonNode.path(DATAPLANE_IP).asText()));
} catch (IllegalArgumentException | NullPointerException e) {
log.error("Failed to read {}", e.toString());
}
});
return nodeMap;
}
/**
* Returns physical router mac.
......@@ -38,7 +73,7 @@ public class OpenstackRoutingConfig extends Config<ApplicationId> {
* @return physical router mac
*/
public String physicalRouterMac() {
return this.get("physicalRouterMac", "");
return this.get(PHYSICAL_ROUTER_MAC, "");
}
/**
......@@ -47,7 +82,7 @@ public class OpenstackRoutingConfig extends Config<ApplicationId> {
* @return bridge id
*/
public String gatewayBridgeId() {
return this.get("gatewayBridgeId", "");
return this.get(GATEWAY_BRIDGE_ID, "");
}
/**
......@@ -56,7 +91,7 @@ public class OpenstackRoutingConfig extends Config<ApplicationId> {
* @return external interface name
*/
public String gatewayExternalInterfaceName() {
return this.get("gatewayExternalInterfaceName", "");
return this.get(GATEWAY_EXTERNAL_INTERFACE_NAME, "");
}
/**
......@@ -65,6 +100,6 @@ public class OpenstackRoutingConfig extends Config<ApplicationId> {
* @return external interface mac
*/
public String gatewayExternalInterfaceMac() {
return this.get("gatewayExternalInterfaceMac", "");
return this.get(GATEWAY_EXTERNAL_INTERFACE_MAC, "");
}
}
......
......@@ -62,6 +62,8 @@ import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
......@@ -108,6 +110,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected StorageService storageService;
private ApplicationId appId;
private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
private ConsistentMap<String, OpenstackFloatingIP> floatingIpMap; // Map<FloatingIp`s Id, FloatingIp object>
......@@ -117,6 +120,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
private static final String FLOATING_IP_MAP_NAME = "openstackrouting-floatingip";
private static final String TP_PORT_MAP_NAME = "openstackrouting-portnum";
private static final String COLON = ":";
private static final int PNAT_PORT_EXPIRE_TIME = 1200 * 1000;
private static final int TP_PORT_MINIMUM_NUM = 1024;
private static final int TP_PORT_MAXIMUM_NUM = 65535;
......@@ -154,6 +158,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
private OpenstackIcmpHandler openstackIcmpHandler;
private OpenstackRoutingArpHandler openstackArpHandler;
private OpenstackRoutingRulePopulator rulePopulator;
private Map<DeviceId, Ip4Address> computeNodeMap;
@Activate
protected void activate() {
......@@ -314,7 +319,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
}
private void reloadInitL3Rules() {
l3EventExecutorService.submit(() ->
l3EventExecutorService.execute(() ->
openstackService.ports()
.stream()
.filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
......@@ -359,7 +364,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
switch (iPacket.getProtocol()) {
case IPv4.PROTOCOL_ICMP:
icmpEventExecutorService.submit(() ->
icmpEventExecutorService.execute(() ->
openstackIcmpHandler.processIcmpPacket(context, ethernet));
break;
case IPv4.PROTOCOL_UDP:
......@@ -371,20 +376,21 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
}
default:
int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
Port port =
Optional<Port> port =
getExternalPort(pkt.receivedFrom().deviceId(), config.gatewayExternalInterfaceName());
if (port == null) {
if (!port.isPresent()) {
log.warn("There`s no external interface");
break;
} else {
OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
Ip4Address.valueOf(iPacket.getSourceAddress()));
l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
portNum, openstackPort, port.get(), config));
}
OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
Ip4Address.valueOf(iPacket.getSourceAddress()));
l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
portNum, openstackPort, port, config));
break;
}
} else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
arpEventExecutorService.submit(() ->
arpEventExecutorService.execute(() ->
openstackArpHandler.processArpPacketFromRouter(context, ethernet));
}
}
......@@ -395,7 +401,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
clearPortNumMap();
portNum = findUnusedPortNum();
}
tpPortNumMap.put(portNum, sourceMac.toString().concat(":").concat(String.valueOf(destinationAddress)));
tpPortNumMap.put(portNum, sourceMac.toString().concat(COLON).concat(String.valueOf(destinationAddress)));
return portNum;
}
......@@ -418,12 +424,11 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
});
}
private Port getExternalPort(DeviceId deviceId, String interfaceName) {
private Optional<Port> getExternalPort(DeviceId deviceId, String interfaceName) {
return deviceService.getPorts(deviceId)
.stream()
.filter(p -> p.annotations().value(PORT_NAME).equals(interfaceName))
.findAny()
.orElse(null);
.findAny();
}
private void checkExternalConnection(OpenstackRouter router,
......@@ -436,33 +441,26 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
log.debug("Not satisfied to set pnat configuration");
return;
}
if (router.id() == null) {
interfaces.forEach(i -> initiateL3Rule(getRouterfromExternalIp(externalIp), i));
} else {
interfaces.forEach(i -> initiateL3Rule(router, i));
}
interfaces.forEach(this::initiateL3Rule);
}
private OpenstackRouter getRouterfromExternalIp(Ip4Address externalIp) {
OpenstackRouter router = getExternalRouter(true)
private Optional<OpenstackRouter> getRouterfromExternalIp(Ip4Address externalIp) {
return getExternalRouter(true)
.stream()
.filter(r -> r.gatewayExternalInfo()
.externalFixedIps()
.values()
.stream()
.findFirst()
.orElse(null)
.findAny()
.get()
.equals(externalIp))
.findAny()
.orElse(null);
return checkNotNull(router);
.findAny();
}
private void initiateL3Rule(OpenstackRouter router, OpenstackRouterInterface routerInterface) {
private void initiateL3Rule(OpenstackRouterInterface routerInterface) {
long vni = Long.parseLong(openstackService.network(openstackService
.port(routerInterface.portId()).networkId()).segmentId());
rulePopulator.populateExternalRules(vni, router, routerInterface);
rulePopulator.populateExternalRules(vni);
}
private Collection<OpenstackRouterInterface> getOpenstackRouterInterface(OpenstackRouter router) {
......@@ -472,8 +470,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
.filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
.filter(p -> p.deviceId().equals(router.id()))
.forEach(p -> {
OpenstackRouterInterface routerInterface = portToRouterInterface(p);
interfaces.add(routerInterface);
interfaces.add(portToRouterInterface(p));
});
return interfaces;
}
......@@ -529,9 +526,6 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
if (event.type().equals(NetworkConfigEvent.Type.CONFIG_ADDED) ||
event.type().equals(NetworkConfigEvent.Type.CONFIG_UPDATED)) {
l3EventExecutorService.execute(OpenstackRoutingManager.this::readConfiguration);
rulePopulator = new OpenstackRoutingRulePopulator(appId,
openstackService, flowObjectiveService, deviceService, driverService, config);
}
}
}
......
......@@ -49,7 +49,6 @@ import org.onosproject.net.flowobjective.ForwardingObjective;
import org.onosproject.net.packet.InboundPacket;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstackinterface.OpenstackRouter;
import org.onosproject.openstackinterface.OpenstackRouterInterface;
import org.onosproject.openstackinterface.OpenstackSubnet;
import org.onosproject.openstackinterface.OpenstackFloatingIP;
......@@ -82,11 +81,11 @@ public class OpenstackRoutingRulePopulator {
private static final String PORTNOTNULL = "Port can not be null";
private static final String DEVICENOTNULL = "Device can not be null";
private static final String TUNNEL_DESTINATION = "tunnelDst";
private static final String DEVICE_ANNOTATION_CHANNELID = "channelId";
private static final int ROUTING_RULE_PRIORITY = 25000;
private static final int FLOATING_RULE_PRIORITY = 42000;
private static final int PNAT_RULE_PRIORITY = 26000;
private static final int PNAT_TIMEOUT = 120;
private static final int PREFIX_LENGTH = 32;
private static final MacAddress GATEWAYMAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f");
private InboundPacket inboundPacket;
......@@ -94,8 +93,6 @@ public class OpenstackRoutingRulePopulator {
private int portNum;
private MacAddress externalInterface;
private MacAddress externalRouter;
private OpenstackRouter router;
private OpenstackRouterInterface routerInterface;
/**
* The constructor of openstackRoutingRulePopulator.
......@@ -247,7 +244,7 @@ public class OpenstackRoutingRulePopulator {
private Ip4Address getHostIpfromOpenstackPort(OpenstackPort openstackPort) {
Device device = getDevicefromOpenstackPort(openstackPort);
return getIPAddressforDevice(device);
return config.nodes().get(device.id());
}
private Device getDevicefromOpenstackPort(OpenstackPort openstackPort) {
......@@ -266,7 +263,7 @@ public class OpenstackRoutingRulePopulator {
.filter(p -> p.isEnabled() && p.annotations().value(PORTNAME).equals(openstackPortName))
.findAny()
.orElse(null);
return port != null ? true : false;
return port != null;
}
/**
......@@ -317,13 +314,8 @@ public class OpenstackRoutingRulePopulator {
* Populates flow rules from openstackComputeNode to GatewayNode.
*
* @param vni Target network
* @param router corresponding router
* @param routerInterface corresponding routerInterface
*/
public void populateExternalRules(long vni, OpenstackRouter router,
OpenstackRouterInterface routerInterface) {
this.router = router;
this.routerInterface = routerInterface;
public void populateExternalRules(long vni) {
// 1. computeNode to gateway
populateComputeNodeRules(vni);
......@@ -367,7 +359,7 @@ public class OpenstackRoutingRulePopulator {
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchTunnelId(vni)
.matchEthDst(GATEWAYMAC);
tBuilder.extension(buildNiciraExtenstion(d.id(), getIPAddressforDevice(gatewayDevice)), d.id())
tBuilder.extension(buildNiciraExtenstion(d.id(), config.nodes().get(gatewayDevice.id())), d.id())
.setOutput(getTunnelPort(d.id()));
ForwardingObjective fo = DefaultForwardingObjective.builder()
......@@ -381,10 +373,6 @@ public class OpenstackRoutingRulePopulator {
flowObjectiveService.forward(d.id(), fo);
}
private Ip4Address getIPAddressforDevice(Device device) {
return Ip4Address.valueOf(device.annotations().value(DEVICE_ANNOTATION_CHANNELID).split(":")[0]);
}
private Device getGatewayNode() {
return checkNotNull(deviceService.getDevice(DeviceId.deviceId(config.gatewayBridgeId())));
}
......@@ -411,11 +399,10 @@ public class OpenstackRoutingRulePopulator {
StreamSupport.stream(deviceService.getDevices().spliterator(), false)
.forEach(d -> {
if (checkGatewayNode(d.id())) {
removeRule(d.id(), sBuilder, ForwardingObjective.Flag.VERSATILE, ROUTING_RULE_PRIORITY);
} else {
removeRule(d.id(), sBuilder, ForwardingObjective.Flag.SPECIFIC, ROUTING_RULE_PRIORITY);
}
ForwardingObjective.Flag flag = checkGatewayNode(d.id()) ?
ForwardingObjective.Flag.VERSATILE : ForwardingObjective.Flag.SPECIFIC;
removeRule(d.id(), sBuilder, flag, ROUTING_RULE_PRIORITY);
});
}
......@@ -457,13 +444,14 @@ public class OpenstackRoutingRulePopulator {
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), 32));
.matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
tBuilder.setEthSrc(MacAddress.valueOf(config.gatewayExternalInterfaceMac()))
.setEthDst(port.macAddress())
.setIpDst(floatingIP.fixedIpAddress())
.setTunnelId(getVni(port.networkId()))
.extension(buildNiciraExtenstion(gatewayNode.id(), getIPAddressforDevice(portNode)), gatewayNode.id())
.extension(buildNiciraExtenstion(gatewayNode.id(),
config.nodes().get(portNode.id())), gatewayNode.id())
.setOutput(getTunnelPort(gatewayNode.id()));
ForwardingObjective fo = DefaultForwardingObjective.builder()
......@@ -516,10 +504,10 @@ public class OpenstackRoutingRulePopulator {
sOutgoingBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchTunnelId(portInfo.vni())
.matchIPSrc(IpPrefix.valueOf(portInfo.ip(), 32));
.matchIPSrc(IpPrefix.valueOf(portInfo.ip(), PREFIX_LENGTH));
sIncomingBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), 32));
.matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
removeRule(getGatewayNode().id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
removeRule(getGatewayNode().id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
......
{
"apps" : {
"org.onosproject.openstackswitching" : {
"openstackswitching" : {
"do_not_push_flows" : "false",
"neutron_server" : "http://10.40.101.209:9696/v2.0/",
"keystone_server" : "http://10.40.101.209:5000/v2.0/",
"user_name" : "admin",
"password" : "nova"
}
},
"org.onosproject.dhcp" : {
"dhcp" : {
"ip": "10.0.0.1",
"mac": "1a:2b:3c:4e:5e:6f",
"subnet": "255.0.0.0",
"broadcast": "10.255.255.255",
"router": "10.0.0.1",
"domain": "10.0.0.1",
"ttl": "63",
"lease": "300",
"renew": "150",
"rebind": "200",
"delay": "3",
"timeout": "150",
"startip": "10.0.0.110",
"endip": "10.0.0.130"
}
},
"org.onosproject.cordvtn" : {
"cordvtn" : {
"nodes" : [
{
"hostname" : "compute-01",
"ovsdbIp" : "10.40.101.208",
"ovsdbPort" : "6640",
"bridgeId" : "of:0000000000000001"
},
{
"hostname" : "compute-02",
"ovsdbIp" : "10.40.101.227",
"ovsdbPort" : "6640",
"bridgeId" : "of:0000000000000002"
},
{
"hostname" : "network",
"ovsdbIp" : "10.40.101.209",
"ovsdbPort" : "6640",
"bridgeId" : "of:0000000000000003"
}
]
}
}
},
"devices" : {
"of:0000000000000001" : {
"basic" : {
"driver" : "sona"
}
},
"of:0000000000000002" : {
"basic" : {
"driver" : "sona"
}
},
"of:0000000000000003" : {
"basic" : {
"driver" : "sona"
}
}
}
}
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.openstacknetworking.switching;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Maps;
import org.onlab.packet.Ip4Address;
import org.onosproject.core.ApplicationId;
import org.onosproject.net.DeviceId;
import org.onosproject.net.config.Config;
import org.slf4j.Logger;
import java.util.Map;
import static org.slf4j.LoggerFactory.getLogger;
/**
* Configuration object for OpenstackSwitching service.
*/
public class OpenstackSwitchingConfig extends Config<ApplicationId> {
protected final Logger log = getLogger(getClass());
public static final String NODES = "nodes";
public static final String DATAPLANE_IP = "dataPlaneIp";
public static final String BRIDGE_ID = "bridgeId";
/**
* Returns the data plane IP map of nodes read from network config.
*
* @return data plane IP map
*/
public Map<DeviceId, Ip4Address> nodes() {
Map<DeviceId, Ip4Address> nodeMap = Maps.newHashMap();
JsonNode jsonNodes = object.get(NODES);
if (jsonNodes == null) {
log.error("There's no node information");
return null;
}
jsonNodes.forEach(jsonNode -> {
try {
nodeMap.putIfAbsent(DeviceId.deviceId(jsonNode.path(BRIDGE_ID).asText()),
Ip4Address.valueOf(jsonNode.path(DATAPLANE_IP).asText()));
} catch (IllegalArgumentException | NullPointerException e) {
log.error("Failed to read {}", e.getMessage());
}
});
return nodeMap;
}
}
......@@ -15,8 +15,8 @@
*/
package org.onosproject.openstacknetworking.switching;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
......@@ -32,8 +32,13 @@ import org.onosproject.dhcp.DhcpService;
import org.onosproject.event.AbstractEvent;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Host;
import org.onosproject.net.Port;
import org.onosproject.net.config.ConfigFactory;
import org.onosproject.net.config.NetworkConfigEvent;
import org.onosproject.net.config.NetworkConfigListener;
import org.onosproject.net.config.NetworkConfigRegistry;
import org.onosproject.net.config.NetworkConfigService;
import org.onosproject.net.config.basics.SubjectFactories;
import org.onosproject.net.device.DeviceEvent;
import org.onosproject.net.device.DeviceListener;
import org.onosproject.net.device.DeviceService;
......@@ -70,8 +75,7 @@ import static org.onlab.util.Tools.groupedThreads;
*/
public class OpenstackSwitchingManager implements OpenstackSwitchingService {
private final Logger log = LoggerFactory
.getLogger(getClass());
private final Logger log = LoggerFactory.getLogger(getClass());
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected CoreService coreService;
......@@ -97,28 +101,52 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected OpenstackInterfaceService openstackService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected NetworkConfigService configService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected NetworkConfigRegistry configRegistry;
public static final String PORTNAME_PREFIX_VM = "tap";
public static final String PORTNAME_PREFIX_ROUTER = "qr-";
public static final String PORTNAME_PREFIX_TUNNEL = "vxlan";
public static final String PORTNAME = "portName";
private static final String ROUTER_INTERFACE = "network:router_interface";
public static final String DEVICE_OWNER_GATEWAY = "network:router_gateway";
public static final String DNS_SERVER_IP = "8.8.8.8";
private static final String FORWARD_SLASH = "/";
private ApplicationId appId;
private OpenstackArpHandler arpHandler;
private OpenstackSecurityGroupRulePopulator sgRulePopulator;
private ExecutorService deviceEventExcutorService =
private ExecutorService deviceEventExecutorService =
Executors.newSingleThreadExecutor(groupedThreads("onos/openstackswitching", "device-event"));
private ExecutorService configEventExecutorService =
Executors.newSingleThreadExecutor(groupedThreads("onos/openstackswitching", "config-event"));
private InternalPacketProcessor internalPacketProcessor = new InternalPacketProcessor();
private InternalDeviceListener internalDeviceListener = new InternalDeviceListener();
private InternalHostListener internalHostListener = new InternalHostListener();
private Map<String, OpenstackPortInfo> openstackPortInfoMap = Maps.newHashMap();
private final Map<String, OpenstackPortInfo> openstackPortInfoMap = Maps.newHashMap();
private Map<String, OpenstackSecurityGroup> securityGroupMap = Maps.newConcurrentMap();
private final ConfigFactory configFactory =
new ConfigFactory(SubjectFactories.APP_SUBJECT_FACTORY,
OpenstackSwitchingConfig.class, "openstackswitching") {
@Override
public OpenstackSwitchingConfig createConfig() {
return new OpenstackSwitchingConfig();
}
};
private final NetworkConfigListener configListener = new InternalConfigListener();
private OpenstackSwitchingConfig config;
@Activate
protected void activate() {
appId = coreService
......@@ -127,12 +155,10 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
packetService.addProcessor(internalPacketProcessor, PacketProcessor.director(1));
deviceService.addListener(internalDeviceListener);
hostService.addListener(internalHostListener);
arpHandler = new OpenstackArpHandler(openstackService, packetService, hostService);
configRegistry.registerConfigFactory(configFactory);
configService.addListener(configListener);
arpHandler = new OpenstackArpHandler(openstackService, packetService, hostService);
sgRulePopulator = new OpenstackSecurityGroupRulePopulator(appId, openstackService, flowObjectiveService);
initializeFlowRules();
readConfiguration();
log.info("Started");
}
......@@ -142,7 +168,11 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
packetService.removeProcessor(internalPacketProcessor);
deviceService.removeListener(internalDeviceListener);
deviceEventExcutorService.shutdown();
deviceEventExecutorService.shutdown();
configEventExecutorService.shutdown();
hostService.removeListener(internalHostListener);
configService.removeListener(configListener);
configRegistry.unregisterConfigFactory(configFactory);
log.info("Stopped");
}
......@@ -151,10 +181,9 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
public void createPorts(OpenstackPort openstackPort) {
if (!openstackPort.deviceOwner().equals(ROUTER_INTERFACE)
&& !openstackPort.deviceOwner().equals(DEVICE_OWNER_GATEWAY)) {
if (!openstackPort.fixedIps().isEmpty()) {
&& !openstackPort.deviceOwner().equals(DEVICE_OWNER_GATEWAY)
&& !openstackPort.fixedIps().isEmpty()) {
registerDhcpInfo(openstackPort);
}
}
}
......@@ -170,11 +199,11 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
if (routerPortInfo != null) {
dhcpService.removeStaticMapping(routerPortInfo.mac());
deviceService.getPorts(routerPortInfo.deviceId()).forEach(port -> {
String pName = port.annotations().value("portName");
String pName = port.annotations().value(PORTNAME);
if (pName.equals(routerPortName)) {
OpenstackSwitchingRulePopulator rulePopulator =
new OpenstackSwitchingRulePopulator(appId, flowObjectiveService,
deviceService, openstackService, driverService);
deviceService, openstackService, driverService, config);
rulePopulator.removeSwitchingRules(port, openstackPortInfoMap);
openstackPortInfoMap.remove(routerPortName);
......@@ -219,10 +248,6 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
return ImmutableMap.copyOf(this.openstackPortInfoMap);
}
private void processDeviceAdded(Device device) {
log.debug("device {} is added", device.id());
}
private void processPortUpdated(Device device, Port port) {
String portName = port.annotations().value(PORTNAME);
synchronized (openstackPortInfoMap) {
......@@ -230,7 +255,7 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
if (port.isEnabled()) {
OpenstackSwitchingRulePopulator rulePopulator =
new OpenstackSwitchingRulePopulator(appId, flowObjectiveService,
deviceService, openstackService, driverService);
deviceService, openstackService, driverService, config);
rulePopulator.populateSwitchingRules(device, port);
OpenstackPort openstackPort = rulePopulator.openstackPort(port);
......@@ -244,10 +269,10 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
//In case portupdate event is driven by vm shutoff from openstack
} else if (!port.isEnabled() && openstackPortInfoMap.containsKey(portName)) {
log.debug("Flowrules according to the port {} were removed", port.number().toString());
log.debug("Flowrules according to the port {} were removed", port.number());
OpenstackSwitchingRulePopulator rulePopulator =
new OpenstackSwitchingRulePopulator(appId, flowObjectiveService,
deviceService, openstackService, driverService);
deviceService, openstackService, driverService, config);
rulePopulator.removeSwitchingRules(port, openstackPortInfoMap);
openstackPortInfoMap.get(portName).securityGroups().stream().forEach(
sgId -> sgRulePopulator.removeSecurityGroupRules(device.id(), sgId,
......@@ -259,14 +284,14 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
}
}
private void processPortRemoved(Device device, Port port) {
private void processPortRemoved(Port port) {
log.debug("port {} is removed", port.toString());
}
private void initializeFlowRules() {
OpenstackSwitchingRulePopulator rulePopulator =
new OpenstackSwitchingRulePopulator(appId, flowObjectiveService,
deviceService, openstackService, driverService);
deviceService, openstackService, driverService, config);
Collection<OpenstackNetwork> networks = openstackService.networks();
Collection<OpenstackSubnet> subnets = openstackService.subnets();
......@@ -336,16 +361,8 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
});
}
private void processHostRemoved(Host host) {
log.debug("host {} was removed", host.toString());
}
private void registerDhcpInfo(OpenstackPort openstackPort) {
Ip4Address ip4Address;
Ip4Address subnetMask;
Ip4Address gatewayIPAddress;
Ip4Address dhcpServer;
Ip4Address domainServer;
Ip4Address ip4Address, subnetMask, gatewayIPAddress, dhcpServer, domainServer;
OpenstackSubnet openstackSubnet;
ip4Address = (Ip4Address) openstackPort.fixedIps().values().stream().findFirst().orElse(null);
......@@ -359,22 +376,18 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
dhcpServer = gatewayIPAddress;
// TODO: supports multiple DNS servers
if (openstackSubnet.dnsNameservers().isEmpty()) {
domainServer = Ip4Address.valueOf("8.8.8.8");
domainServer = Ip4Address.valueOf(DNS_SERVER_IP);
} else {
domainServer = openstackSubnet.dnsNameservers().get(0);
}
List<Ip4Address> options = Lists.newArrayList();
options.add(subnetMask);
options.add(dhcpServer);
options.add(gatewayIPAddress);
options.add(domainServer);
List<Ip4Address> options = ImmutableList.of(subnetMask, dhcpServer, gatewayIPAddress, domainServer);
dhcpService.setStaticMapping(openstackPort.macAddress(), ip4Address, true, options);
}
private byte[] buildSubnetMask(String cidr) {
int prefix;
String[] parts = cidr.split("/");
String[] parts = cidr.split(FORWARD_SLASH);
prefix = Integer.parseInt(parts[1]);
int mask = 0xffffffff << (32 - prefix);
byte[] bytes = new byte[]{(byte) (mask >>> 24),
......@@ -406,7 +419,7 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
@Override
public void event(HostEvent hostEvent) {
deviceEventExcutorService.execute(new InternalEventHandler(hostEvent));
deviceEventExecutorService.execute(new InternalEventHandler(hostEvent));
}
}
......@@ -414,7 +427,7 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
@Override
public void event(DeviceEvent deviceEvent) {
deviceEventExcutorService.execute(new InternalEventHandler(deviceEvent));
deviceEventExecutorService.execute(new InternalEventHandler(deviceEvent));
}
}
......@@ -434,22 +447,22 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
switch (deviceEvent.type()) {
case DEVICE_ADDED:
processDeviceAdded((Device) deviceEvent.subject());
log.debug("device {} is added", deviceEvent.subject().id());
break;
case DEVICE_AVAILABILITY_CHANGED:
Device device = (Device) deviceEvent.subject();
Device device = deviceEvent.subject();
if (deviceService.isAvailable(device.id())) {
processDeviceAdded(device);
log.debug("device {} is added", device.id());
}
break;
case PORT_ADDED:
processPortUpdated((Device) deviceEvent.subject(), deviceEvent.port());
processPortUpdated(deviceEvent.subject(), deviceEvent.port());
break;
case PORT_UPDATED:
processPortUpdated((Device) deviceEvent.subject(), deviceEvent.port());
processPortUpdated(deviceEvent.subject(), deviceEvent.port());
break;
case PORT_REMOVED:
processPortRemoved((Device) deviceEvent.subject(), deviceEvent.port());
processPortRemoved(deviceEvent.port());
break;
default:
log.debug("Unsupported deviceEvent type {}", deviceEvent.type().toString());
......@@ -460,7 +473,7 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
switch (hostEvent.type()) {
case HOST_REMOVED:
processHostRemoved((Host) hostEvent.subject());
log.debug("host {} was removed", hostEvent.subject().toString());
break;
default:
log.debug("Unsupported hostEvent type {}", hostEvent.type().toString());
......@@ -469,4 +482,34 @@ public class OpenstackSwitchingManager implements OpenstackSwitchingService {
}
}
}
private void readConfiguration() {
config = configService.getConfig(appId, OpenstackSwitchingConfig.class);
if (config == null) {
log.error("No configuration found");
return;
}
arpHandler = new OpenstackArpHandler(openstackService, packetService, hostService);
sgRulePopulator = new OpenstackSecurityGroupRulePopulator(appId, openstackService, flowObjectiveService);
initializeFlowRules();
}
private class InternalConfigListener implements NetworkConfigListener {
@Override
public void event(NetworkConfigEvent event) {
if (!event.configClass().equals(OpenstackSwitchingConfig.class)) {
return;
}
if (event.type().equals(NetworkConfigEvent.Type.CONFIG_ADDED) ||
event.type().equals(NetworkConfigEvent.Type.CONFIG_UPDATED)) {
configEventExecutorService.execute(OpenstackSwitchingManager.this::readConfiguration);
}
}
}
}
......
......@@ -59,11 +59,13 @@ public class OpenstackSwitchingRulePopulator {
.getLogger(OpenstackSwitchingRulePopulator.class);
private static final int SWITCHING_RULE_PRIORITY = 30000;
private static final int TUNNELTAG_RULE_PRIORITY = 30000;
private static final String PORT_NAME = "portName";
private static final String TUNNEL_DST = "tunnelDst";
private FlowObjectiveService flowObjectiveService;
private DriverService driverService;
private DeviceService deviceService;
private ApplicationId appId;
private OpenstackSwitchingConfig config;
private Collection<OpenstackNetwork> openstackNetworkList;
private Collection<OpenstackPort> openstackPortList;
......@@ -76,16 +78,19 @@ public class OpenstackSwitchingRulePopulator {
* @param deviceService DeviceService reference
* @param openstackService openstack interface service
* @param driverService DriverService reference
* @param config OpenstackRoutingConfig
*/
public OpenstackSwitchingRulePopulator(ApplicationId appId,
FlowObjectiveService flowObjectiveService,
DeviceService deviceService,
OpenstackInterfaceService openstackService,
DriverService driverService) {
DriverService driverService,
OpenstackSwitchingConfig config) {
this.flowObjectiveService = flowObjectiveService;
this.deviceService = deviceService;
this.driverService = driverService;
this.appId = appId;
this.config = config;
openstackNetworkList = openstackService.networks();
openstackPortList = openstackService.ports();
......@@ -111,8 +116,8 @@ public class OpenstackSwitchingRulePopulator {
* @param port port info of the VM
*/
private void populateFlowRulesForTunnelTag(Device device, Port port) {
Ip4Address vmIp = getFixedIpAddressForPort(port.annotations().value("portName"));
String portName = port.annotations().value("portName");
Ip4Address vmIp = getFixedIpAddressForPort(port.annotations().value(PORT_NAME));
String portName = port.annotations().value(PORT_NAME);
String vni = getVniForPort(portName);
if (vmIp != null) {
......@@ -148,8 +153,8 @@ public class OpenstackSwitchingRulePopulator {
* @param port port info of the VM
*/
private void populateFlowRulesForTrafficToSameCnode(Device device, Port port) {
Ip4Address vmIp = getFixedIpAddressForPort(port.annotations().value("portName"));
String portName = port.annotations().value("portName");
Ip4Address vmIp = getFixedIpAddressForPort(port.annotations().value(PORT_NAME));
String portName = port.annotations().value(PORT_NAME);
String vni = getVniForPort(portName);
if (vmIp != null) {
......@@ -197,24 +202,27 @@ public class OpenstackSwitchingRulePopulator {
* @param port port information of the VM
*/
private void populateFlowRulesForTrafficToDifferentCnode(Device device, Port port) {
String portName = port.annotations().value("portName");
String channelId = device.annotations().value("channelId");
Ip4Address hostIpAddress = Ip4Address.valueOf(channelId.split(":")[0]);
String portName = port.annotations().value(PORT_NAME);
Ip4Address fixedIp = getFixedIpAddressForPort(portName);
String vni = getVniForPort(portName);
Ip4Address hostDpIpAddress = config.nodes().get(device.id());
if (hostDpIpAddress == null) {
log.debug("There's no openstack node information for device id {}", device.id().toString());
return;
}
deviceService.getAvailableDevices().forEach(d -> {
if (!d.equals(device)) {
deviceService.getPorts(d.id()).forEach(p -> {
String pName = p.annotations().value("portName");
String pName = p.annotations().value(PORT_NAME);
if (!p.equals(port) && vni.equals(getVniForPort(pName))) {
String cidx = d.annotations().value("channelId");
Ip4Address hostIpx = Ip4Address.valueOf(cidx.split(":")[0]);
Ip4Address hostxDpIpAddress = config.nodes().get(d.id());
Ip4Address fixedIpx = getFixedIpAddressForPort(pName);
if (port.isEnabled() ||
port.annotations().value("portName").startsWith(
OpenstackSwitchingManager.PORTNAME_PREFIX_ROUTER)) {
setVxLanFlowRule(vni, device.id(), hostIpx, fixedIpx);
setVxLanFlowRule(vni, d.id(), hostIpAddress, fixedIp);
if (port.isEnabled()) {
setVxLanFlowRule(vni, device.id(), hostxDpIpAddress, fixedIpx);
setVxLanFlowRule(vni, d.id(), hostDpIpAddress, fixedIp);
}
}
});
......@@ -259,7 +267,7 @@ public class OpenstackSwitchingRulePopulator {
* @return OpenstackPort reference, or null when not found
*/
public OpenstackPort openstackPort(Port port) {
String uuid = port.annotations().value("portName").substring(3);
String uuid = port.annotations().value(PORT_NAME).substring(3);
return openstackPortList.stream().filter(p -> p.id().startsWith(uuid))
.findAny().orElse(null);
}
......@@ -273,7 +281,7 @@ public class OpenstackSwitchingRulePopulator {
public void removeSwitchingRules(Port removedPort, Map<String,
OpenstackPortInfo> openstackPortInfoMap) {
OpenstackPortInfo openstackPortInfo = openstackPortInfoMap
.get(removedPort.annotations().value("portName"));
.get(removedPort.annotations().value(PORT_NAME));
DeviceId deviceId = openstackPortInfo.deviceId();
Ip4Address vmIp = openstackPortInfo.ip();
......@@ -397,7 +405,7 @@ public class OpenstackSwitchingRulePopulator {
Map<String, OpenstackPortInfo> openstackPortInfoMap) {
for (Map.Entry<String, OpenstackPortInfo> entry : openstackPortInfoMap.entrySet()) {
if (!removedPort.annotations().value("portName").equals(entry.getKey())) {
if (!removedPort.annotations().value(PORT_NAME).equals(entry.getKey())) {
if (entry.getValue().vni() == vni && entry.getValue().deviceId().equals(deviceId)) {
return true;
}
......@@ -469,7 +477,7 @@ public class OpenstackSwitchingRulePopulator {
ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST.type());
try {
extensionInstruction.setPropertyValue("tunnelDst", hostIp);
extensionInstruction.setPropertyValue(TUNNEL_DST, hostIp);
} catch (ExtensionPropertyException e) {
log.error("Error setting Nicira extension setting {}", e);
}
......@@ -479,7 +487,7 @@ public class OpenstackSwitchingRulePopulator {
private PortNumber getTunnelPort(DeviceId deviceId) {
Port port = deviceService.getPorts(deviceId).stream()
.filter(p -> p.annotations().value("portName").equals(
.filter(p -> p.annotations().value(PORT_NAME).equals(
OpenstackSwitchingManager.PORTNAME_PREFIX_TUNNEL))
.findAny().orElse(null);
......