Marc De Leenheer
Committed by Gerrit Code Review

Injecting topology through JSON ConfigProvider works for multi-instance (ONOS-490).

Change-Id: Ib977f4cf9a59ddec360072891fd803c6f9ee84f1

Injecting optical device annotations and ports works for multi-instance (ONOS-870).

Change-Id: Icdde16ef72fc4e47eec7213250b04902083f0537
......@@ -15,10 +15,10 @@
*/
package org.onosproject.event;
import static com.google.common.base.MoreObjects.toStringHelper;
import org.joda.time.LocalDateTime;
import static com.google.common.base.MoreObjects.toStringHelper;
/**
* Base event implementation.
*/
......@@ -75,5 +75,4 @@ public class AbstractEvent<T extends Enum, S> implements Event<T, S> {
.add("subject", subject())
.toString();
}
}
......
......@@ -47,7 +47,7 @@ public class DefaultDeviceDescription extends AbstractDescription
* @param hwVersion device HW version
* @param swVersion device SW version
* @param serialNumber device serial number
* @param chassis chasis id
* @param chassis chassis id
* @param annotations optional key/value annotations map
*/
public DefaultDeviceDescription(URI uri, Type type, String manufacturer,
......
......@@ -306,18 +306,16 @@ public class DeviceManager
// TODO: Do we need to explicitly tell the Provider that
// this instance is not the MASTER
applyRole(deviceId, MastershipRole.STANDBY);
return;
}
} else {
log.info("Role of this node is MASTER for {}", deviceId);
// tell clock provider if this instance is the master
deviceClockProviderService.setMastershipTerm(deviceId, term);
applyRole(deviceId, MastershipRole.MASTER);
}
DeviceEvent event = store.createOrUpdateDevice(provider().id(),
deviceId, deviceDescription);
applyRole(deviceId, MastershipRole.MASTER);
// If there was a change of any kind, tell the provider
// that this instance is the master.
if (event != null) {
......
package org.onosproject.store.device.impl;
import com.google.common.base.MoreObjects;
import org.onosproject.net.DeviceId;
import org.onosproject.net.device.DeviceDescription;
import org.onosproject.net.provider.ProviderId;
public class DeviceInjectedEvent {
private final ProviderId providerId;
private final DeviceId deviceId;
private final DeviceDescription deviceDescription;
protected DeviceInjectedEvent(
ProviderId providerId,
DeviceId deviceId,
DeviceDescription deviceDescription) {
this.providerId = providerId;
this.deviceId = deviceId;
this.deviceDescription = deviceDescription;
}
public DeviceId deviceId() {
return deviceId;
}
public ProviderId providerId() {
return providerId;
}
public DeviceDescription deviceDescription() {
return deviceDescription;
}
@Override
public String toString() {
return MoreObjects.toStringHelper(getClass())
.add("providerId", providerId)
.add("deviceId", deviceId)
.add("deviceDescription", deviceDescription)
.toString();
}
// for serializer
protected DeviceInjectedEvent() {
this.providerId = null;
this.deviceId = null;
this.deviceDescription = null;
}
}
......@@ -21,7 +21,6 @@ import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.RandomUtils;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
......@@ -29,6 +28,9 @@ import org.apache.felix.scr.annotations.Deactivate;
import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.apache.felix.scr.annotations.Service;
import org.onlab.packet.ChassisId;
import org.onlab.util.KryoNamespace;
import org.onlab.util.NewConcurrentHashMap;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.ControllerNode;
import org.onosproject.cluster.NodeId;
......@@ -61,9 +63,6 @@ import org.onosproject.store.cluster.messaging.MessageSubject;
import org.onosproject.store.impl.Timestamped;
import org.onosproject.store.serializers.KryoSerializer;
import org.onosproject.store.serializers.impl.DistributedStoreSerializers;
import org.onlab.packet.ChassisId;
import org.onlab.util.KryoNamespace;
import org.onlab.util.NewConcurrentHashMap;
import org.slf4j.Logger;
import java.io.IOException;
......@@ -86,17 +85,17 @@ import java.util.concurrent.TimeUnit;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Predicates.notNull;
import static org.onosproject.cluster.ControllerNodeToNodeId.toNodeId;
import static org.onosproject.net.device.DeviceEvent.Type.*;
import static org.slf4j.LoggerFactory.getLogger;
import static org.apache.commons.lang3.concurrent.ConcurrentUtils.createIfAbsentUnchecked;
import static org.onosproject.net.DefaultAnnotations.merge;
import static com.google.common.base.Verify.verify;
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
import static org.apache.commons.lang3.concurrent.ConcurrentUtils.createIfAbsentUnchecked;
import static org.onlab.util.Tools.minPriority;
import static org.onlab.util.Tools.namedThreads;
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
import static org.onosproject.store.device.impl.GossipDeviceStoreMessageSubjects.DEVICE_ADVERTISE;
import static org.onosproject.store.device.impl.GossipDeviceStoreMessageSubjects.DEVICE_REMOVE_REQ;
import static org.onosproject.cluster.ControllerNodeToNodeId.toNodeId;
import static org.onosproject.net.DefaultAnnotations.merge;
import static org.onosproject.net.device.DeviceEvent.Type.*;
import static org.onosproject.net.device.DeviceEvent.Type.DEVICE_REMOVED;
import static org.onosproject.store.device.impl.GossipDeviceStoreMessageSubjects.*;
import static org.slf4j.LoggerFactory.getLogger;
/**
* Manages inventory of infrastructure devices using gossip protocol to distribute
......@@ -111,6 +110,8 @@ public class GossipDeviceStore
private final Logger log = getLogger(getClass());
private static final String DEVICE_NOT_FOUND = "Device with ID %s not found";
// Timeout in milliseconds to process device or ports on remote master node
private static final int REMOTE_MASTER_TIMEOUT = 1000;
// innerMap is used to lock a Device, thus instance should never be replaced.
// collection of Description given from various providers
......@@ -158,6 +159,8 @@ public class GossipDeviceStore
.register(DeviceAntiEntropyAdvertisement.class)
.register(DeviceFragmentId.class)
.register(PortFragmentId.class)
.register(DeviceInjectedEvent.class)
.register(PortInjectedEvent.class)
.build();
}
};
......@@ -186,6 +189,10 @@ public class GossipDeviceStore
GossipDeviceStoreMessageSubjects.PORT_STATUS_UPDATE, new InternalPortStatusEventListener());
clusterCommunicator.addSubscriber(
GossipDeviceStoreMessageSubjects.DEVICE_ADVERTISE, new InternalDeviceAdvertisementListener());
clusterCommunicator.addSubscriber(
GossipDeviceStoreMessageSubjects.DEVICE_INJECTED, new DeviceInjectedEventListener());
clusterCommunicator.addSubscriber(
GossipDeviceStoreMessageSubjects.PORT_INJECTED, new PortInjectedEventListener());
executor = Executors.newCachedThreadPool(namedThreads("onos-device-fg-%d"));
......@@ -251,21 +258,48 @@ public class GossipDeviceStore
public synchronized DeviceEvent createOrUpdateDevice(ProviderId providerId,
DeviceId deviceId,
DeviceDescription deviceDescription) {
NodeId localNode = clusterService.getLocalNode().id();
NodeId deviceNode = mastershipService.getMasterFor(deviceId);
// Process device update only if we're the master,
// otherwise signal the actual master.
DeviceEvent deviceEvent = null;
if (localNode.equals(deviceNode)) {
final Timestamp newTimestamp = deviceClockService.getTimestamp(deviceId);
final Timestamped<DeviceDescription> deltaDesc = new Timestamped<>(deviceDescription, newTimestamp);
final DeviceEvent event;
final Timestamped<DeviceDescription> mergedDesc;
final Map<ProviderId, DeviceDescriptions> device = getOrCreateDeviceDescriptionsMap(deviceId);
synchronized (device) {
event = createOrUpdateDeviceInternal(providerId, deviceId, deltaDesc);
deviceEvent = createOrUpdateDeviceInternal(providerId, deviceId, deltaDesc);
mergedDesc = device.get(providerId).getDeviceDesc();
}
if (event != null) {
if (deviceEvent != null) {
log.info("Notifying peers of a device update topology event for providerId: {} and deviceId: {}",
providerId, deviceId);
notifyPeers(new InternalDeviceEvent(providerId, deviceId, mergedDesc));
}
return event;
} else {
DeviceInjectedEvent deviceInjectedEvent = new DeviceInjectedEvent(
providerId, deviceId, deviceDescription);
ClusterMessage clusterMessage = new ClusterMessage(localNode, DEVICE_INJECTED,
SERIALIZER.encode(deviceInjectedEvent));
try {
clusterCommunicator.unicast(clusterMessage, deviceNode);
} catch (IOException e) {
log.warn("Failed to process injected device id: {} desc: {} " +
"(cluster messaging failed: {})",
deviceId, deviceDescription, e);
}
}
return deviceEvent;
}
private DeviceEvent createOrUpdateDeviceInternal(ProviderId providerId,
......@@ -434,6 +468,19 @@ public class GossipDeviceStore
DeviceId deviceId,
List<PortDescription> portDescriptions) {
NodeId localNode = clusterService.getLocalNode().id();
// TODO: It might be negligible, but this will have negative impact to topology discovery performance,
// since it will trigger distributed store read.
// Also, it'll probably be better if side-way communication happened on ConfigurationProvider, etc.
// outside Device subsystem. so that we don't have to modify both Device and Link stores.
// If we don't care much about topology performance, then it might be OK.
NodeId deviceNode = mastershipService.getMasterFor(deviceId);
// Process port update only if we're the master of the device,
// otherwise signal the actual master.
List<DeviceEvent> deviceEvents = null;
if (localNode.equals(deviceNode)) {
final Timestamp newTimestamp;
try {
newTimestamp = deviceClockService.getTimestamp(deviceId);
......@@ -456,12 +503,12 @@ public class GossipDeviceStore
final Timestamped<List<PortDescription>> timestampedInput
= new Timestamped<>(portDescriptions, newTimestamp);
final List<DeviceEvent> events;
final Timestamped<List<PortDescription>> merged;
final Map<ProviderId, DeviceDescriptions> device = getOrCreateDeviceDescriptionsMap(deviceId);
synchronized (device) {
events = updatePortsInternal(providerId, deviceId, timestampedInput);
deviceEvents = updatePortsInternal(providerId, deviceId, timestampedInput);
final DeviceDescriptions descs = device.get(providerId);
List<PortDescription> mergedList =
FluentIterable.from(portDescriptions)
......@@ -474,12 +521,28 @@ public class GossipDeviceStore
}).toList();
merged = new Timestamped<List<PortDescription>>(mergedList, newTimestamp);
}
if (!events.isEmpty()) {
if (!deviceEvents.isEmpty()) {
log.info("Notifying peers of a ports update topology event for providerId: {} and deviceId: {}",
providerId, deviceId);
notifyPeers(new InternalPortEvent(providerId, deviceId, merged));
}
return events;
} else {
PortInjectedEvent portInjectedEvent = new PortInjectedEvent(providerId, deviceId, portDescriptions);
ClusterMessage clusterMessage = new ClusterMessage(
localNode, PORT_INJECTED, SERIALIZER.encode(portInjectedEvent));
try {
clusterCommunicator.unicast(clusterMessage, deviceNode);
} catch (IOException e) {
log.warn("Failed to process injected ports of device id: {} " +
"(cluster messaging failed: {})",
deviceId, e);
}
}
return deviceEvents;
}
private List<DeviceEvent> updatePortsInternal(ProviderId providerId,
......@@ -1431,4 +1494,48 @@ public class GossipDeviceStore
});
}
}
private final class DeviceInjectedEventListener
implements ClusterMessageHandler {
@Override
public void handle(ClusterMessage message) {
log.debug("Received injected device event from peer: {}", message.sender());
DeviceInjectedEvent event = SERIALIZER.decode(message.payload());
ProviderId providerId = event.providerId();
DeviceId deviceId = event.deviceId();
DeviceDescription deviceDescription = event.deviceDescription();
executor.submit(new Runnable() {
@Override
public void run() {
createOrUpdateDevice(providerId, deviceId, deviceDescription);
}
});
}
}
private final class PortInjectedEventListener
implements ClusterMessageHandler {
@Override
public void handle(ClusterMessage message) {
log.debug("Received injected port event from peer: {}", message.sender());
PortInjectedEvent event = SERIALIZER.decode(message.payload());
ProviderId providerId = event.providerId();
DeviceId deviceId = event.deviceId();
List<PortDescription> portDescriptions = event.portDescriptions();
executor.submit(new Runnable() {
@Override
public void run() {
updatePorts(providerId, deviceId, portDescriptions);
}
});
}
}
}
......
......@@ -34,4 +34,8 @@ public final class GossipDeviceStoreMessageSubjects {
public static final MessageSubject DEVICE_ADVERTISE = new MessageSubject("peer-device-advertisements");
// to be used with 3-way anti-entropy process
public static final MessageSubject DEVICE_REQUEST = new MessageSubject("peer-device-request");
// Network elements injected (not discovered) by ConfigProvider
public static final MessageSubject DEVICE_INJECTED = new MessageSubject("peer-device-injected");
public static final MessageSubject PORT_INJECTED = new MessageSubject("peer-port-injected");
}
......
package org.onosproject.store.device.impl;
import com.google.common.base.MoreObjects;
import org.onosproject.net.DeviceId;
import org.onosproject.net.device.PortDescription;
import org.onosproject.net.provider.ProviderId;
import java.util.List;
public class PortInjectedEvent {
private ProviderId providerId;
private DeviceId deviceId;
private List<PortDescription> portDescriptions;
protected PortInjectedEvent(ProviderId providerId, DeviceId deviceId, List<PortDescription> portDescriptions) {
this.providerId = providerId;
this.deviceId = deviceId;
this.portDescriptions = portDescriptions;
}
public DeviceId deviceId() {
return deviceId;
}
public ProviderId providerId() {
return providerId;
}
public List<PortDescription> portDescriptions() {
return portDescriptions;
}
@Override
public String toString() {
return MoreObjects.toStringHelper(getClass())
.add("providerId", providerId)
.add("deviceId", deviceId)
.add("portDescriptions", portDescriptions)
.toString();
}
// for serializer
protected PortInjectedEvent() {
this.providerId = null;
this.deviceId = null;
this.portDescriptions = null;
}
}
......@@ -32,6 +32,7 @@ import org.onlab.util.KryoNamespace;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.ControllerNode;
import org.onosproject.cluster.NodeId;
import org.onosproject.mastership.MastershipService;
import org.onosproject.net.AnnotationKeys;
import org.onosproject.net.AnnotationsUtil;
import org.onosproject.net.ConnectPoint;
......@@ -90,9 +91,7 @@ import static org.onosproject.net.Link.State.INACTIVE;
import static org.onosproject.net.Link.Type.DIRECT;
import static org.onosproject.net.Link.Type.INDIRECT;
import static org.onosproject.net.LinkKey.linkKey;
import static org.onosproject.net.link.LinkEvent.Type.LINK_ADDED;
import static org.onosproject.net.link.LinkEvent.Type.LINK_REMOVED;
import static org.onosproject.net.link.LinkEvent.Type.LINK_UPDATED;
import static org.onosproject.net.link.LinkEvent.Type.*;
import static org.onosproject.store.link.impl.GossipLinkStoreMessageSubjects.LINK_ANTI_ENTROPY_ADVERTISEMENT;
import static org.slf4j.LoggerFactory.getLogger;
......@@ -106,6 +105,9 @@ public class GossipLinkStore
extends AbstractStore<LinkEvent, LinkStoreDelegate>
implements LinkStore {
// Timeout in milliseconds to process links on remote master node
private static final int REMOTE_MASTER_TIMEOUT = 1000;
private final Logger log = getLogger(getClass());
// Link inventory
......@@ -131,6 +133,9 @@ public class GossipLinkStore
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ClusterService clusterService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected MastershipService mastershipService;
protected static final KryoSerializer SERIALIZER = new KryoSerializer() {
@Override
protected void setupKryoPool() {
......@@ -141,6 +146,7 @@ public class GossipLinkStore
.register(InternalLinkRemovedEvent.class)
.register(LinkAntiEntropyAdvertisement.class)
.register(LinkFragmentId.class)
.register(LinkInjectedEvent.class)
.build();
}
};
......@@ -161,6 +167,9 @@ public class GossipLinkStore
clusterCommunicator.addSubscriber(
GossipLinkStoreMessageSubjects.LINK_ANTI_ENTROPY_ADVERTISEMENT,
new InternalLinkAntiEntropyAdvertisementListener());
clusterCommunicator.addSubscriber(
GossipLinkStoreMessageSubjects.LINK_INJECTED,
new LinkInjectedEventListener());
executor = Executors.newCachedThreadPool(namedThreads("onos-link-fg-%d"));
......@@ -270,27 +279,52 @@ public class GossipLinkStore
public LinkEvent createOrUpdateLink(ProviderId providerId,
LinkDescription linkDescription) {
DeviceId dstDeviceId = linkDescription.dst().deviceId();
final DeviceId dstDeviceId = linkDescription.dst().deviceId();
final NodeId localNode = clusterService.getLocalNode().id();
final NodeId dstNode = mastershipService.getMasterFor(dstDeviceId);
// Process link update only if we're the master of the destination node,
// otherwise signal the actual master.
LinkEvent linkEvent = null;
if (localNode.equals(dstNode)) {
Timestamp newTimestamp = deviceClockService.getTimestamp(dstDeviceId);
final Timestamped<LinkDescription> deltaDesc = new Timestamped<>(linkDescription, newTimestamp);
LinkKey key = linkKey(linkDescription.src(), linkDescription.dst());
final LinkEvent event;
final Timestamped<LinkDescription> mergedDesc;
Map<ProviderId, Timestamped<LinkDescription>> map = getOrCreateLinkDescriptions(key);
synchronized (map) {
event = createOrUpdateLinkInternal(providerId, deltaDesc);
linkEvent = createOrUpdateLinkInternal(providerId, deltaDesc);
mergedDesc = map.get(providerId);
}
if (event != null) {
if (linkEvent != null) {
log.info("Notifying peers of a link update topology event from providerId: "
+ "{} between src: {} and dst: {}",
providerId, linkDescription.src(), linkDescription.dst());
notifyPeers(new InternalLinkEvent(providerId, mergedDesc));
}
return event;
} else {
LinkInjectedEvent linkInjectedEvent = new LinkInjectedEvent(providerId, linkDescription);
ClusterMessage linkInjectedMessage = new ClusterMessage(localNode,
GossipLinkStoreMessageSubjects.LINK_INJECTED, SERIALIZER.encode(linkInjectedEvent));
try {
clusterCommunicator.unicast(linkInjectedMessage, dstNode);
} catch (IOException e) {
log.warn("Failed to process link update between src: {} and dst: {} " +
"(cluster messaging failed: {})",
linkDescription.src(), linkDescription.dst(), e);
}
}
return linkEvent;
}
@Override
......@@ -397,7 +431,7 @@ public class GossipLinkStore
!AnnotationsUtil.isEqual(oldLink.annotations(), newLink.annotations())) {
links.put(key, newLink);
// strictly speaking following can be ommitted
// strictly speaking following can be omitted
srcLinks.put(oldLink.src().deviceId(), key);
dstLinks.put(oldLink.dst().deviceId(), key);
return new LinkEvent(LINK_UPDATED, newLink);
......@@ -848,4 +882,25 @@ public class GossipLinkStore
});
}
}
private final class LinkInjectedEventListener
implements ClusterMessageHandler {
@Override
public void handle(ClusterMessage message) {
log.trace("Received injected link event from peer: {}", message.sender());
LinkInjectedEvent linkInjectedEvent = SERIALIZER.decode(message.payload());
ProviderId providerId = linkInjectedEvent.providerId();
LinkDescription linkDescription = linkInjectedEvent.linkDescription();
executor.submit(new Runnable() {
@Override
public void run() {
createOrUpdateLink(providerId, linkDescription);
}
});
}
}
}
......
......@@ -15,7 +15,7 @@
*/
package org.onosproject.store.link.impl;
import org.onosproject.store.cluster.messaging.MessageSubject;
import org.onosproject.store.cluster.messaging.MessageSubject;
/**
* MessageSubjects used by GossipLinkStore peer-peer communication.
......@@ -30,4 +30,6 @@ public final class GossipLinkStoreMessageSubjects {
new MessageSubject("peer-link-removed");
public static final MessageSubject LINK_ANTI_ENTROPY_ADVERTISEMENT =
new MessageSubject("link-enti-entropy-advertisement");
public static final MessageSubject LINK_INJECTED =
new MessageSubject("peer-link-injected");
}
......
package org.onosproject.store.link.impl;
import com.google.common.base.MoreObjects;
import org.onosproject.net.link.LinkDescription;
import org.onosproject.net.provider.ProviderId;
public class LinkInjectedEvent {
ProviderId providerId;
LinkDescription linkDescription;
public LinkInjectedEvent(ProviderId providerId, LinkDescription linkDescription) {
this.providerId = providerId;
this.linkDescription = linkDescription;
}
public ProviderId providerId() {
return providerId;
}
public LinkDescription linkDescription() {
return linkDescription;
}
@Override
public String toString() {
return MoreObjects.toStringHelper(getClass())
.add("providerId", providerId)
.add("linkDescription", linkDescription)
.toString();
}
// for serializer
protected LinkInjectedEvent() {
this.providerId = null;
this.linkDescription = null;
}
}
......@@ -27,6 +27,8 @@ import org.onlab.packet.IpAddress;
import org.onosproject.cluster.ControllerNode;
import org.onosproject.cluster.DefaultControllerNode;
import org.onosproject.cluster.NodeId;
import org.onosproject.mastership.MastershipService;
import org.onosproject.mastership.MastershipServiceAdapter;
import org.onosproject.mastership.MastershipTerm;
import org.onosproject.net.ConnectPoint;
import org.onosproject.net.DefaultAnnotations;
......@@ -115,7 +117,7 @@ public class GossipLinkStoreTest {
private DeviceClockManager deviceClockManager;
private DeviceClockService deviceClockService;
private ClusterCommunicationService clusterCommunicator;
private MastershipService mastershipService;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
......@@ -146,11 +148,13 @@ public class GossipLinkStoreTest {
linkStoreImpl.deviceClockService = deviceClockService;
linkStoreImpl.clusterCommunicator = clusterCommunicator;
linkStoreImpl.clusterService = new TestClusterService();
linkStoreImpl.mastershipService = new TestMastershipService();
linkStoreImpl.activate();
linkStore = linkStoreImpl;
verify(clusterCommunicator);
reset(clusterCommunicator);
}
@After
......@@ -602,4 +606,11 @@ public class GossipLinkStoreTest {
nodeStates.put(NID2, ACTIVE);
}
}
private final class TestMastershipService extends MastershipServiceAdapter {
@Override
public NodeId getMasterFor(DeviceId deviceId) {
return NID1;
}
}
}
......