홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Madan Jampani
2af244a9 1 parent a853709a

Never process incoming messages on the netty event loop thread pool. 

Currently in a lot of places we are deserializing incoming messages on this threadpool and that could be significantly limiting throughput.

Change-Id: I83eb7e91004cea4addb28bc28f27e50de10028fe
Inline Side-by-side
Showing 18 changed files with 325 additions and 259 deletions
......@@ -16,10 +16,12 @@
package org.onosproject.store.cluster.messaging;
import com.google.common.util.concurrent.ListenableFuture;
import org.onosproject.cluster.NodeId;
import java.io.IOException;
import java.util.Set;
import java.util.concurrent.ExecutorService;
// TODO: remove IOExceptions?
/**
......@@ -77,9 +79,19 @@ public interface ClusterCommunicationService {
* @param subject message subject
* @param subscriber message subscriber
*/
@Deprecated
void addSubscriber(MessageSubject subject, ClusterMessageHandler subscriber);
/**
* Adds a new subscriber for the specified message subject.
*
* @param subject message subject
* @param subscriber message subscriber
* @param executor executor to use for running handler.
*/
void addSubscriber(MessageSubject subject, ClusterMessageHandler subscriber, ExecutorService executor);
/**
* Removes a subscriber for the specified message subject.
*
* @param subject message subject
......
......@@ -18,6 +18,7 @@ package org.onosproject.store.app;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.ListenableFuture;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
......@@ -55,6 +56,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
......@@ -90,6 +92,8 @@ public class GossipApplicationStore extends ApplicationArchive
private final ScheduledExecutorService executor =
Executors.newSingleThreadScheduledExecutor(groupedThreads("onos/app", "store"));
private ExecutorService messageHandlingExecutor;
private EventuallyConsistentMap<ApplicationId, Application> apps;
private EventuallyConsistentMap<Application, InternalState> states;
private EventuallyConsistentMap<Application, Set<Permission>> permissions;
......@@ -109,7 +113,10 @@ public class GossipApplicationStore extends ApplicationArchive
.register(KryoNamespaces.API)
.register(InternalState.class);
clusterCommunicator.addSubscriber(APP_BITS_REQUEST, new InternalBitServer());
messageHandlingExecutor = Executors.newSingleThreadExecutor(
groupedThreads("onos/store/app", "message-handler"));
clusterCommunicator.addSubscriber(APP_BITS_REQUEST, new InternalBitServer(), messageHandlingExecutor);
apps = new EventuallyConsistentMapImpl<>("apps", clusterService,
clusterCommunicator,
......@@ -145,6 +152,8 @@ public class GossipApplicationStore extends ApplicationArchive
@Deactivate
public void deactivate() {
clusterCommunicator.removeSubscriber(APP_BITS_REQUEST);
messageHandlingExecutor.shutdown();
apps.destroy();
states.destroy();
permissions.destroy();
......
......@@ -18,6 +18,7 @@ package org.onosproject.store.cluster.impl;
import com.google.common.collect.Maps;
import com.hazelcast.config.TopicConfig;
import com.hazelcast.core.IAtomicLong;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
......@@ -113,6 +114,8 @@ public class HazelcastLeadershipService implements LeadershipService {
private static final MessageSubject LEADERSHIP_EVENT_MESSAGE_SUBJECT =
new MessageSubject("hz-leadership-events");
private ExecutorService messageHandlingExecutor;
@Activate
protected void activate() {
localNodeId = clusterService.getLocalNode().id();
......@@ -124,7 +127,13 @@ public class HazelcastLeadershipService implements LeadershipService {
topicConfig.setName(TOPIC_HZ_ID);
storeService.getHazelcastInstance().getConfig().addTopicConfig(topicConfig);
clusterCommunicator.addSubscriber(LEADERSHIP_EVENT_MESSAGE_SUBJECT, new InternalLeadershipEventListener());
messageHandlingExecutor = Executors.newSingleThreadExecutor(
groupedThreads("onos/store/leadership", "message-handler"));
clusterCommunicator.addSubscriber(
LEADERSHIP_EVENT_MESSAGE_SUBJECT,
new InternalLeadershipEventListener(),
messageHandlingExecutor);
log.info("Hazelcast Leadership Service started");
}
......@@ -132,6 +141,7 @@ public class HazelcastLeadershipService implements LeadershipService {
@Deactivate
protected void deactivate() {
eventDispatcher.removeSink(LeadershipEvent.class);
messageHandlingExecutor.shutdown();
clusterCommunicator.removeSubscriber(LEADERSHIP_EVENT_MESSAGE_SUBJECT);
for (Topic topic : topics.values()) {
......
......@@ -18,6 +18,7 @@ package org.onosproject.store.cluster.impl;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
......@@ -43,6 +44,7 @@ import org.slf4j.Logger;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
......@@ -92,6 +94,8 @@ public class LeadershipManager implements LeadershipService {
private final LeadershipEventListener peerAdvertiser = new PeerAdvertiser();
private final LeadershipEventListener leaderBoardUpdater = new LeaderBoardUpdater();
private ExecutorService messageHandlingExecutor;
public static final KryoSerializer SERIALIZER = new KryoSerializer() {
@Override
protected void setupKryoPool() {
......@@ -109,9 +113,14 @@ public class LeadershipManager implements LeadershipService {
addListener(peerAdvertiser);
addListener(leaderBoardUpdater);
messageHandlingExecutor = Executors.newSingleThreadExecutor(
groupedThreads("onos/store/leadership",
"peer-advertisement-handler"));
clusterCommunicator.addSubscriber(
LEADERSHIP_UPDATES,
new PeerAdvertisementHandler());
new PeerAdvertisementHandler(),
messageHandlingExecutor);
log.info("Started.");
}
......@@ -123,6 +132,7 @@ public class LeadershipManager implements LeadershipService {
clusterCommunicator.removeSubscriber(LEADERSHIP_UPDATES);
messageHandlingExecutor.shutdown();
threadPool.shutdown();
log.info("Stopped.");
......
......@@ -16,6 +16,7 @@
package org.onosproject.store.cluster.messaging.impl;
import com.google.common.util.concurrent.ListenableFuture;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
......@@ -44,6 +45,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import static com.google.common.base.Preconditions.checkArgument;
......@@ -183,6 +185,13 @@ public class ClusterCommunicationManager
}
@Override
public void addSubscriber(MessageSubject subject,
ClusterMessageHandler subscriber,
ExecutorService executor) {
messagingService.registerHandler(subject.value(), new InternalClusterMessageHandler(subscriber), executor);
}
@Override
public void removeSubscriber(MessageSubject subject) {
messagingService.unregisterHandler(subject.value());
}
......
......@@ -176,28 +176,35 @@ public class GossipDeviceStore
@Activate
public void activate() {
executor = Executors.newCachedThreadPool(groupedThreads("onos/device", "fg-%d"));
backgroundExecutor =
newSingleThreadScheduledExecutor(minPriority(groupedThreads("onos/device", "bg-%d")));
clusterCommunicator.addSubscriber(
GossipDeviceStoreMessageSubjects.DEVICE_UPDATE, new InternalDeviceEventListener());
GossipDeviceStoreMessageSubjects.DEVICE_UPDATE, new InternalDeviceEventListener(), executor);
clusterCommunicator.addSubscriber(
GossipDeviceStoreMessageSubjects.DEVICE_OFFLINE, new InternalDeviceOfflineEventListener());
clusterCommunicator.addSubscriber(DEVICE_REMOVE_REQ, new InternalRemoveRequestListener());
GossipDeviceStoreMessageSubjects.DEVICE_OFFLINE,
new InternalDeviceOfflineEventListener(),
executor);
clusterCommunicator.addSubscriber(DEVICE_REMOVE_REQ,
new InternalRemoveRequestListener(),
executor);
clusterCommunicator.addSubscriber(
GossipDeviceStoreMessageSubjects.DEVICE_REMOVED, new InternalDeviceRemovedEventListener());
GossipDeviceStoreMessageSubjects.DEVICE_REMOVED, new InternalDeviceRemovedEventListener(), executor);
clusterCommunicator.addSubscriber(
GossipDeviceStoreMessageSubjects.PORT_UPDATE, new InternalPortEventListener());
GossipDeviceStoreMessageSubjects.PORT_UPDATE, new InternalPortEventListener(), executor);
clusterCommunicator.addSubscriber(
GossipDeviceStoreMessageSubjects.PORT_STATUS_UPDATE, new InternalPortStatusEventListener());
GossipDeviceStoreMessageSubjects.PORT_STATUS_UPDATE, new InternalPortStatusEventListener(), executor);
clusterCommunicator.addSubscriber(
GossipDeviceStoreMessageSubjects.DEVICE_ADVERTISE, new InternalDeviceAdvertisementListener());
GossipDeviceStoreMessageSubjects.DEVICE_ADVERTISE,
new InternalDeviceAdvertisementListener(),
backgroundExecutor);
clusterCommunicator.addSubscriber(
GossipDeviceStoreMessageSubjects.DEVICE_INJECTED, new DeviceInjectedEventListener());
GossipDeviceStoreMessageSubjects.DEVICE_INJECTED, new DeviceInjectedEventListener(), executor);
clusterCommunicator.addSubscriber(
GossipDeviceStoreMessageSubjects.PORT_INJECTED, new PortInjectedEventListener());
executor = Executors.newCachedThreadPool(groupedThreads("onos/device", "fg-%d"));
backgroundExecutor =
newSingleThreadScheduledExecutor(minPriority(groupedThreads("onos/device", "bg-%d")));
GossipDeviceStoreMessageSubjects.PORT_INJECTED, new PortInjectedEventListener(), executor);
// start anti-entropy thread
backgroundExecutor.scheduleAtFixedRate(new SendAdvertisementTask(),
......@@ -1325,17 +1332,11 @@ public class GossipDeviceStore
DeviceId deviceId = event.deviceId();
Timestamped<DeviceDescription> deviceDescription = event.deviceDescription();
executor.submit(new Runnable() {
@Override
public void run() {
try {
notifyDelegateIfNotNull(createOrUpdateDeviceInternal(providerId, deviceId, deviceDescription));
} catch (Exception e) {
log.warn("Exception thrown handling device update", e);
}
}
});
try {
notifyDelegateIfNotNull(createOrUpdateDeviceInternal(providerId, deviceId, deviceDescription));
} catch (Exception e) {
log.warn("Exception thrown handling device update", e);
}
}
}
......@@ -1350,17 +1351,11 @@ public class GossipDeviceStore
DeviceId deviceId = event.deviceId();
Timestamp timestamp = event.timestamp();
executor.submit(new Runnable() {
@Override
public void run() {
try {
notifyDelegateIfNotNull(markOfflineInternal(deviceId, timestamp));
} catch (Exception e) {
log.warn("Exception thrown handling device offline", e);
}
}
});
try {
notifyDelegateIfNotNull(markOfflineInternal(deviceId, timestamp));
} catch (Exception e) {
log.warn("Exception thrown handling device offline", e);
}
}
}
......@@ -1371,17 +1366,11 @@ public class GossipDeviceStore
log.debug("Received device remove request from peer: {}", message.sender());
DeviceId did = SERIALIZER.decode(message.payload());
executor.submit(new Runnable() {
@Override
public void run() {
try {
removeDevice(did);
} catch (Exception e) {
log.warn("Exception thrown handling device remove", e);
}
}
});
try {
removeDevice(did);
} catch (Exception e) {
log.warn("Exception thrown handling device remove", e);
}
}
}
......@@ -1396,17 +1385,11 @@ public class GossipDeviceStore
DeviceId deviceId = event.deviceId();
Timestamp timestamp = event.timestamp();
executor.submit(new Runnable() {
@Override
public void run() {
try {
notifyDelegateIfNotNull(removeDeviceInternal(deviceId, timestamp));
} catch (Exception e) {
log.warn("Exception thrown handling device removed", e);
}
}
});
try {
notifyDelegateIfNotNull(removeDeviceInternal(deviceId, timestamp));
} catch (Exception e) {
log.warn("Exception thrown handling device removed", e);
}
}
}
......@@ -1428,17 +1411,11 @@ public class GossipDeviceStore
return;
}
executor.submit(new Runnable() {
@Override
public void run() {
try {
notifyDelegate(updatePortsInternal(providerId, deviceId, portDescriptions));
} catch (Exception e) {
log.warn("Exception thrown handling port update", e);
}
}
});
try {
notifyDelegate(updatePortsInternal(providerId, deviceId, portDescriptions));
} catch (Exception e) {
log.warn("Exception thrown handling port update", e);
}
}
}
......@@ -1460,17 +1437,11 @@ public class GossipDeviceStore
return;
}
executor.submit(new Runnable() {
@Override
public void run() {
try {
notifyDelegateIfNotNull(updatePortStatusInternal(providerId, deviceId, portDescription));
} catch (Exception e) {
log.warn("Exception thrown handling port update", e);
}
}
});
try {
notifyDelegateIfNotNull(updatePortStatusInternal(providerId, deviceId, portDescription));
} catch (Exception e) {
log.warn("Exception thrown handling port update", e);
}
}
}
......@@ -1481,17 +1452,11 @@ public class GossipDeviceStore
public void handle(ClusterMessage message) {
log.trace("Received Device Anti-Entropy advertisement from peer: {}", message.sender());
DeviceAntiEntropyAdvertisement advertisement = SERIALIZER.decode(message.payload());
backgroundExecutor.submit(new Runnable() {
@Override
public void run() {
try {
handleAdvertisement(advertisement);
} catch (Exception e) {
log.warn("Exception thrown handling Device advertisements.", e);
}
}
});
try {
handleAdvertisement(advertisement);
} catch (Exception e) {
log.warn("Exception thrown handling Device advertisements.", e);
}
}
}
......@@ -1507,13 +1472,11 @@ public class GossipDeviceStore
DeviceId deviceId = event.deviceId();
DeviceDescription deviceDescription = event.deviceDescription();
executor.submit(new Runnable() {
@Override
public void run() {
createOrUpdateDevice(providerId, deviceId, deviceDescription);
}
});
try {
createOrUpdateDevice(providerId, deviceId, deviceDescription);
} catch (Exception e) {
log.warn("Exception thrown handling device injected event.", e);
}
}
}
......@@ -1529,13 +1492,11 @@ public class GossipDeviceStore
DeviceId deviceId = event.deviceId();
List<PortDescription> portDescriptions = event.portDescriptions();
executor.submit(new Runnable() {
@Override
public void run() {
updatePorts(providerId, deviceId, portDescriptions);
}
});
try {
updatePorts(providerId, deviceId, portDescriptions);
} catch (Exception e) {
log.warn("Exception thrown handling port injected event.", e);
}
}
}
}
......
......@@ -162,13 +162,13 @@ public class EventuallyConsistentMapImpl<K, V>
updateMessageSubject = new MessageSubject("ecm-" + mapName + "-update");
clusterCommunicator.addSubscriber(updateMessageSubject,
new InternalPutEventListener());
new InternalPutEventListener(), executor);
removeMessageSubject = new MessageSubject("ecm-" + mapName + "-remove");
clusterCommunicator.addSubscriber(removeMessageSubject,
new InternalRemoveEventListener());
new InternalRemoveEventListener(), executor);
antiEntropyAdvertisementSubject = new MessageSubject("ecm-" + mapName + "-anti-entropy");
clusterCommunicator.addSubscriber(antiEntropyAdvertisementSubject,
new InternalAntiEntropyListener());
new InternalAntiEntropyListener(), backgroundExecutor);
}
private KryoSerializer createSerializer(KryoNamespace.Builder builder) {
......@@ -728,13 +728,11 @@ public class EventuallyConsistentMapImpl<K, V>
log.trace("Received anti-entropy advertisement from peer: {}",
message.sender());
AntiEntropyAdvertisement<K> advertisement = serializer.decode(message.payload());
backgroundExecutor.submit(() -> {
try {
handleAntiEntropyAdvertisement(advertisement);
} catch (Exception e) {
log.warn("Exception thrown handling advertisements", e);
}
});
try {
handleAntiEntropyAdvertisement(advertisement);
} catch (Exception e) {
log.warn("Exception thrown handling advertisements", e);
}
}
}
......@@ -745,25 +743,23 @@ public class EventuallyConsistentMapImpl<K, V>
log.debug("Received put event from peer: {}", message.sender());
InternalPutEvent<K, V> event = serializer.decode(message.payload());
executor.submit(() -> {
try {
for (PutEntry<K, V> entry : event.entries()) {
K key = entry.key();
V value = entry.value();
Timestamp timestamp = entry.timestamp();
if (putInternal(key, value, timestamp)) {
EventuallyConsistentMapEvent<K, V> externalEvent =
new EventuallyConsistentMapEvent<>(
EventuallyConsistentMapEvent.Type.PUT, key,
value);
notifyListeners(externalEvent);
}
try {
for (PutEntry<K, V> entry : event.entries()) {
K key = entry.key();
V value = entry.value();
Timestamp timestamp = entry.timestamp();
if (putInternal(key, value, timestamp)) {
EventuallyConsistentMapEvent<K, V> externalEvent =
new EventuallyConsistentMapEvent<>(
EventuallyConsistentMapEvent.Type.PUT, key,
value);
notifyListeners(externalEvent);
}
} catch (Exception e) {
log.warn("Exception thrown handling put", e);
}
});
} catch (Exception e) {
log.warn("Exception thrown handling put", e);
}
}
}
......@@ -773,25 +769,22 @@ public class EventuallyConsistentMapImpl<K, V>
public void handle(ClusterMessage message) {
log.debug("Received remove event from peer: {}", message.sender());
InternalRemoveEvent<K> event = serializer.decode(message.payload());
try {
for (RemoveEntry<K> entry : event.entries()) {
K key = entry.key();
Timestamp timestamp = entry.timestamp();
executor.submit(() -> {
try {
for (RemoveEntry<K> entry : event.entries()) {
K key = entry.key();
Timestamp timestamp = entry.timestamp();
if (removeInternal(key, timestamp)) {
EventuallyConsistentMapEvent<K, V> externalEvent
= new EventuallyConsistentMapEvent<>(
EventuallyConsistentMapEvent.Type.REMOVE,
key, null);
notifyListeners(externalEvent);
}
if (removeInternal(key, timestamp)) {
EventuallyConsistentMapEvent<K, V> externalEvent
= new EventuallyConsistentMapEvent<>(
EventuallyConsistentMapEvent.Type.REMOVE,
key, null);
notifyListeners(externalEvent);
}
} catch (Exception e) {
log.warn("Exception thrown handling remove", e);
}
});
} catch (Exception e) {
log.warn("Exception thrown handling remove", e);
}
}
}
......
......@@ -107,6 +107,9 @@ public class DistributedFlowRuleStore
private final Logger log = getLogger(getClass());
// TODO: Make configurable.
private static final int MESSAGE_HANDLER_THREAD_POOL_SIZE = 8;
private InternalFlowTable flowTable = new InternalFlowTable();
/*private final ConcurrentMap<DeviceId, ConcurrentMap<FlowId, Set<StoredFlowEntry>>>
......@@ -132,6 +135,7 @@ public class DistributedFlowRuleStore
// Cache of SMaps used for backup data. each SMap contain device flow table
private LoadingCache<DeviceId, SMap<FlowId, ImmutableList<StoredFlowEntry>>> smaps;
private ExecutorService messageHandlingExecutor;
private final ExecutorService backupExecutors =
Executors.newSingleThreadExecutor(groupedThreads("onos/flow", "async-backups"));
......@@ -172,7 +176,11 @@ public class DistributedFlowRuleStore
final NodeId local = clusterService.getLocalNode().id();
clusterCommunicator.addSubscriber(APPLY_BATCH_FLOWS, new OnStoreBatch(local));
messageHandlingExecutor = Executors.newFixedThreadPool(
MESSAGE_HANDLER_THREAD_POOL_SIZE,
groupedThreads("onos/flow", "message-handlers"));
clusterCommunicator.addSubscriber(APPLY_BATCH_FLOWS, new OnStoreBatch(local), messageHandlingExecutor);
clusterCommunicator.addSubscriber(REMOTE_APPLY_COMPLETED, new ClusterMessageHandler() {
@Override
......@@ -181,7 +189,7 @@ public class DistributedFlowRuleStore
log.trace("received completed notification for {}", event);
notifyDelegate(event);
}
});
}, messageHandlingExecutor);
clusterCommunicator.addSubscriber(GET_FLOW_ENTRY, new ClusterMessageHandler() {
......@@ -196,7 +204,7 @@ public class DistributedFlowRuleStore
log.error("Failed to respond back", e);
}
}
});
}, messageHandlingExecutor);
clusterCommunicator.addSubscriber(GET_DEVICE_FLOW_ENTRIES, new ClusterMessageHandler() {
......@@ -211,7 +219,7 @@ public class DistributedFlowRuleStore
log.error("Failed to respond to peer's getFlowEntries request", e);
}
}
});
}, messageHandlingExecutor);
clusterCommunicator.addSubscriber(REMOVE_FLOW_ENTRY, new ClusterMessageHandler() {
......@@ -226,7 +234,7 @@ public class DistributedFlowRuleStore
log.error("Failed to respond back", e);
}
}
});
}, messageHandlingExecutor);
replicaInfoEventListener = new InternalReplicaInfoEventListener();
......@@ -242,6 +250,7 @@ public class DistributedFlowRuleStore
clusterCommunicator.removeSubscriber(GET_FLOW_ENTRY);
clusterCommunicator.removeSubscriber(APPLY_BATCH_FLOWS);
clusterCommunicator.removeSubscriber(REMOTE_APPLY_COMPLETED);
messageHandlingExecutor.shutdown();
replicaInfoManager.removeListener(replicaInfoEventListener);
log.info("Stopped");
}
......@@ -421,7 +430,7 @@ public class DistributedFlowRuleStore
switch (op.operator()) {
case ADD:
entry = new DefaultFlowEntry(op.target());
// always add requested FlowRule

// always add requested FlowRule
// Note: 2 equal FlowEntry may have different treatment
flowTable.remove(entry.deviceId(), entry);
flowTable.add(entry);
......
......@@ -78,6 +78,9 @@ public class DefaultFlowRuleExtRouter
private final Logger log = getLogger(getClass());
// TODO: Make configurable.
private static final int MESSAGE_HANDLER_THREAD_POOL_SIZE = 4;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ReplicaInfoService replicaInfoManager;
......@@ -102,6 +105,8 @@ public class DefaultFlowRuleExtRouter
private final ExecutorService futureListeners = Executors
.newCachedThreadPool(groupedThreads("onos/flow", "store-peer-responders"));
private ExecutorService messageHandlingExecutor;
protected static final StoreSerializer SERIALIZER = new KryoSerializer() {
@Override
protected void setupKryoPool() {
......@@ -120,6 +125,11 @@ public class DefaultFlowRuleExtRouter
@Activate
public void activate() {
messageHandlingExecutor = Executors.newFixedThreadPool(
MESSAGE_HANDLER_THREAD_POOL_SIZE,
groupedThreads("onos/flow", "message-handlers"));
clusterCommunicator.addSubscriber(APPLY_EXTEND_FLOWS,
new ClusterMessageHandler() {
......@@ -141,7 +151,7 @@ public class DefaultFlowRuleExtRouter
}
}, futureListeners);
}
});
}, messageHandlingExecutor);
replicaInfoManager.addListener(replicaInfoEventListener);
......@@ -151,6 +161,7 @@ public class DefaultFlowRuleExtRouter
@Deactivate
public void deactivate() {
clusterCommunicator.removeSubscriber(APPLY_EXTEND_FLOWS);
messageHandlingExecutor.shutdown();
replicaInfoManager.removeListener(replicaInfoEventListener);
log.info("Stopped");
}
......
......@@ -154,20 +154,21 @@ public class GossipHostStore
@Activate
public void activate() {
executor = newCachedThreadPool(groupedThreads("onos/host", "fg-%d"));
backgroundExecutor =
newSingleThreadScheduledExecutor(minPriority(groupedThreads("onos/host", "bg-%d")));
clusterCommunicator.addSubscriber(
HOST_UPDATED_MSG,
new InternalHostEventListener());
new InternalHostEventListener(), executor);
clusterCommunicator.addSubscriber(
HOST_REMOVED_MSG,
new InternalHostRemovedEventListener());
new InternalHostRemovedEventListener(), executor);
clusterCommunicator.addSubscriber(
HOST_ANTI_ENTROPY_ADVERTISEMENT,
new InternalHostAntiEntropyAdvertisementListener());
executor = newCachedThreadPool(groupedThreads("onos/host", "fg-%d"));
backgroundExecutor =
newSingleThreadScheduledExecutor(minPriority(groupedThreads("onos/host", "bg-%d")));
new InternalHostAntiEntropyAdvertisementListener(), backgroundExecutor);
// start anti-entropy thread
backgroundExecutor.scheduleAtFixedRate(new SendAdvertisementTask(),
......@@ -512,20 +513,14 @@ public class GossipHostStore
HostDescription hostDescription = event.hostDescription();
Timestamp timestamp = event.timestamp();
executor.submit(new Runnable() {
@Override
public void run() {
try {
notifyDelegateIfNotNull(createOrUpdateHostInternal(providerId,
hostId,
hostDescription,
timestamp));
} catch (Exception e) {
log.warn("Exception thrown handling host removed", e);
}
}
});
try {
notifyDelegateIfNotNull(createOrUpdateHostInternal(providerId,
hostId,
hostDescription,
timestamp));
} catch (Exception e) {
log.warn("Exception thrown handling host removed", e);
}
}
}
......@@ -540,17 +535,11 @@ public class GossipHostStore
HostId hostId = event.hostId();
Timestamp timestamp = event.timestamp();
executor.submit(new Runnable() {
@Override
public void run() {
try {
notifyDelegateIfNotNull(removeHostInternal(hostId, timestamp));
} catch (Exception e) {
log.warn("Exception thrown handling host removed", e);
}
}
});
try {
notifyDelegateIfNotNull(removeHostInternal(hostId, timestamp));
} catch (Exception e) {
log.warn("Exception thrown handling host removed", e);
}
}
}
......@@ -720,17 +709,11 @@ public class GossipHostStore
public void handle(ClusterMessage message) {
log.trace("Received Host Anti-Entropy advertisement from peer: {}", message.sender());
HostAntiEntropyAdvertisement advertisement = SERIALIZER.decode(message.payload());
backgroundExecutor.submit(new Runnable() {
@Override
public void run() {
try {
handleAntiEntropyAdvertisement(advertisement);
} catch (Exception e) {
log.warn("Exception thrown handling Host advertisements", e);
}
}
});
try {
handleAntiEntropyAdvertisement(advertisement);
} catch (Exception e) {
log.warn("Exception thrown handling Host advertisements", e);
}
}
}
}
......
......@@ -158,23 +158,23 @@ public class GossipLinkStore
@Activate
public void activate() {
executor = Executors.newCachedThreadPool(groupedThreads("onos/link", "fg-%d"));
backgroundExecutors =
newSingleThreadScheduledExecutor(minPriority(groupedThreads("onos/link", "bg-%d")));
clusterCommunicator.addSubscriber(
GossipLinkStoreMessageSubjects.LINK_UPDATE,
new InternalLinkEventListener());
new InternalLinkEventListener(), executor);
clusterCommunicator.addSubscriber(
GossipLinkStoreMessageSubjects.LINK_REMOVED,
new InternalLinkRemovedEventListener());
new InternalLinkRemovedEventListener(), executor);
clusterCommunicator.addSubscriber(
GossipLinkStoreMessageSubjects.LINK_ANTI_ENTROPY_ADVERTISEMENT,
new InternalLinkAntiEntropyAdvertisementListener());
new InternalLinkAntiEntropyAdvertisementListener(), backgroundExecutors);
clusterCommunicator.addSubscriber(
GossipLinkStoreMessageSubjects.LINK_INJECTED,
new LinkInjectedEventListener());
executor = Executors.newCachedThreadPool(groupedThreads("onos/link", "fg-%d"));
backgroundExecutors =
newSingleThreadScheduledExecutor(minPriority(groupedThreads("onos/link", "bg-%d")));
new LinkInjectedEventListener(), executor);
long initialDelaySec = 5;
long periodSec = 5;
......@@ -822,17 +822,11 @@ public class GossipLinkStore
ProviderId providerId = event.providerId();
Timestamped<LinkDescription> linkDescription = event.linkDescription();
executor.submit(new Runnable() {
@Override
public void run() {
try {
notifyDelegateIfNotNull(createOrUpdateLinkInternal(providerId, linkDescription));
} catch (Exception e) {
log.warn("Exception thrown handling link event", e);
}
}
});
try {
notifyDelegateIfNotNull(createOrUpdateLinkInternal(providerId, linkDescription));
} catch (Exception e) {
log.warn("Exception thrown handling link event", e);
}
}
}
......@@ -847,17 +841,11 @@ public class GossipLinkStore
LinkKey linkKey = event.linkKey();
Timestamp timestamp = event.timestamp();
executor.submit(new Runnable() {
@Override
public void run() {
try {
notifyDelegateIfNotNull(removeLinkInternal(linkKey, timestamp));
} catch (Exception e) {
log.warn("Exception thrown handling link removed", e);
}
}
});
try {
notifyDelegateIfNotNull(removeLinkInternal(linkKey, timestamp));
} catch (Exception e) {
log.warn("Exception thrown handling link removed", e);
}
}
}
......@@ -868,18 +856,12 @@ public class GossipLinkStore
public void handle(ClusterMessage message) {
log.trace("Received Link Anti-Entropy advertisement from peer: {}", message.sender());
LinkAntiEntropyAdvertisement advertisement = SERIALIZER.decode(message.payload());
backgroundExecutors.submit(new Runnable() {
@Override
public void run() {
try {
handleAntiEntropyAdvertisement(advertisement);
} catch (Exception e) {
log.warn("Exception thrown while handling Link advertisements", e);
throw e;
}
}
});
try {
handleAntiEntropyAdvertisement(advertisement);
} catch (Exception e) {
log.warn("Exception thrown while handling Link advertisements", e);
throw e;
}
}
}
......@@ -894,13 +876,11 @@ public class GossipLinkStore
ProviderId providerId = linkInjectedEvent.providerId();
LinkDescription linkDescription = linkInjectedEvent.linkDescription();
executor.submit(new Runnable() {
@Override
public void run() {
createOrUpdateLink(providerId, linkDescription);
}
});
try {
createOrUpdateLink(providerId, linkDescription);
} catch (Exception e) {
log.warn("Exception thrown while handling link injected event", e);
}
}
}
}
......
......@@ -15,9 +15,12 @@
*/
package org.onosproject.store.packet.impl;
import static org.onlab.util.Tools.groupedThreads;
import static org.slf4j.LoggerFactory.getLogger;
import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
......@@ -55,6 +58,9 @@ public class DistributedPacketStore
private final Logger log = getLogger(getClass());
// TODO: make this configurable.
private static final int MESSAGE_HANDLER_THREAD_POOL_SIZE = 4;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
private MastershipService mastershipService;
......@@ -77,16 +83,24 @@ public class DistributedPacketStore
}
};
private ExecutorService messageHandlingExecutor;
@Activate
public void activate() {
log.info("Started");
messageHandlingExecutor = Executors.newFixedThreadPool(
MESSAGE_HANDLER_THREAD_POOL_SIZE,
groupedThreads("onos/flow", "message-handlers"));
communicationService.addSubscriber(
PACKET_OUT_SUBJECT, new InternalClusterMessageHandler());
PACKET_OUT_SUBJECT, new InternalClusterMessageHandler(), messageHandlingExecutor);
log.info("Started");
}
@Deactivate
public void deactivate() {
communicationService.removeSubscriber(PACKET_OUT_SUBJECT);
messageHandlingExecutor.shutdown();
log.info("Stopped");
}
......
......@@ -16,6 +16,7 @@
package org.onosproject.store.statistic.impl;
import com.google.common.collect.Sets;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
......@@ -48,11 +49,14 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.store.statistic.impl.StatisticStoreMessageSubjects.GET_CURRENT;
import static org.onosproject.store.statistic.impl.StatisticStoreMessageSubjects.GET_PREVIOUS;
import static org.slf4j.LoggerFactory.getLogger;
......@@ -68,6 +72,9 @@ public class DistributedStatisticStore implements StatisticStore {
private final Logger log = getLogger(getClass());
// TODO: Make configurable.
private static final int MESSAGE_HANDLER_THREAD_POOL_SIZE = 4;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ReplicaInfoService replicaInfoManager;
......@@ -97,10 +104,17 @@ public class DistributedStatisticStore implements StatisticStore {
}
};;
private ExecutorService messageHandlingExecutor;
private static final long STATISTIC_STORE_TIMEOUT_MILLIS = 3000;
@Activate
public void activate() {
messageHandlingExecutor = Executors.newFixedThreadPool(
MESSAGE_HANDLER_THREAD_POOL_SIZE,
groupedThreads("onos/store/statistic", "message-handlers"));
clusterCommunicator.addSubscriber(GET_CURRENT, new ClusterMessageHandler() {
@Override
......@@ -112,7 +126,7 @@ public class DistributedStatisticStore implements StatisticStore {
log.error("Failed to respond back", e);
}
}
});
}, messageHandlingExecutor);
clusterCommunicator.addSubscriber(GET_PREVIOUS, new ClusterMessageHandler() {
......@@ -125,12 +139,15 @@ public class DistributedStatisticStore implements StatisticStore {
log.error("Failed to respond back", e);
}
}
});
}, messageHandlingExecutor);
log.info("Started");
}
@Deactivate
public void deactivate() {
clusterCommunicator.removeSubscriber(GET_PREVIOUS);
clusterCommunicator.removeSubscriber(GET_CURRENT);
messageHandlingExecutor.shutdown();
log.info("Stopped");
}
......
......@@ -17,6 +17,7 @@ package org.onosproject.store.device.impl;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import org.easymock.Capture;
import org.junit.After;
import org.junit.AfterClass;
......@@ -62,6 +63,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import static java.util.Arrays.asList;
......@@ -152,7 +154,7 @@ public class GossipDeviceStoreTest {
clusterCommunicator = createNiceMock(ClusterCommunicationService.class);
clusterCommunicator.addSubscriber(anyObject(MessageSubject.class),
anyObject(ClusterMessageHandler.class));
anyObject(ClusterMessageHandler.class), anyObject(ExecutorService.class));
expectLastCall().anyTimes();
replay(clusterCommunicator);
ClusterService clusterService = new TestClusterService();
......
......@@ -46,6 +46,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
......@@ -129,7 +130,7 @@ public class EventuallyConsistentMapImplTest {
// allows us to get a reference to the map's internal cluster message
// handlers so we can induce events coming in from a peer.
clusterCommunicator.addSubscriber(anyObject(MessageSubject.class),
anyObject(ClusterMessageHandler.class));
anyObject(ClusterMessageHandler.class), anyObject(ExecutorService.class));
expectLastCall().andDelegateTo(new TestClusterCommunicationService()).times(3);
replay(clusterCommunicator);
......@@ -731,6 +732,21 @@ public class EventuallyConsistentMapImplTest {
}
@Override
public void addSubscriber(MessageSubject subject,
ClusterMessageHandler subscriber,
ExecutorService executor) {
if (subject.equals(PUT_MESSAGE_SUBJECT)) {
putHandler = subscriber;
} else if (subject.equals(REMOVE_MESSAGE_SUBJECT)) {
removeHandler = subscriber;
} else if (subject.equals(ANTI_ENTROPY_MESSAGE_SUBJECT)) {
antiEntropyHandler = subscriber;
} else {
throw new RuntimeException("Unexpected message subject " + subject.toString());
}
}
@Override
public void removeSubscriber(MessageSubject subject) {}
}
......
......@@ -16,6 +16,7 @@
package org.onosproject.store.link.impl;
import com.google.common.collect.Iterables;
import org.easymock.Capture;
import org.junit.After;
import org.junit.AfterClass;
......@@ -56,6 +57,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import static org.easymock.EasyMock.*;
......@@ -140,7 +142,8 @@ public class GossipLinkStoreTest {
// TODO mock clusterCommunicator
clusterCommunicator = createNiceMock(ClusterCommunicationService.class);
clusterCommunicator.addSubscriber(anyObject(MessageSubject.class),
anyObject(ClusterMessageHandler.class));
anyObject(ClusterMessageHandler.class),
anyObject(ExecutorService.class));
expectLastCall().anyTimes();
replay(clusterCommunicator);
......
......@@ -16,6 +16,7 @@
package org.onlab.netty;
import java.io.IOException;
import java.util.concurrent.ExecutorService;
import com.google.common.util.concurrent.ListenableFuture;
......@@ -47,7 +48,16 @@ public interface MessagingService {
* Registers a new message handler for message type.
* @param type message type.
* @param handler message handler
* @param executor executor to use for running message handler logic.
*/
public void registerHandler(String type, MessageHandler handler, ExecutorService executor);
/**
* Registers a new message handler for message type.
* @param type message type.
* @param handler message handler
*/
@Deprecated
public void registerHandler(String type, MessageHandler handler);
/**
......
......@@ -41,6 +41,7 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
......@@ -213,6 +214,22 @@ public class NettyMessagingService implements MessagingService {
}
@Override
public void registerHandler(String type, MessageHandler handler, ExecutorService executor) {
handlers.putIfAbsent(hashToLong(type), new MessageHandler() {
@Override
public void handle(Message message) throws IOException {
executor.submit(() -> {
try {
handler.handle(message);
} catch (Exception e) {
log.warn("Failed to process message of type {}", type, e);
}
});
}
});
}
@Override
public void unregisterHandler(String type) {
handlers.remove(type);
}
......