홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Madan Jampani
Committed by Gerrit Code Review
afeebbd5 1 parent 69d5f4e4

ONOS-1982: MessagingService is now a OSGi service. Has implementations based on Netty and IOLoop 

Change-Id: Ia4c99de18e91be1b49bd1fddd86fe89fb83e859c
Expand all
Inline Side-by-side
Showing 19 changed files with 339 additions and 185 deletions
1 +package org.onosproject.cluster;
2 +
3 +import java.util.Set;
4 +
5 +/**
6 + * Service for obtaining the static definition of a controller cluster.
7 + */
8 +public interface ClusterDefinitionService {
9 +
10 + /**
11 + * Returns the local controller node.
12 + * @return local controller node
13 + */
14 + ControllerNode localNode();
15 +
16 + /**
17 + * Returns the set of seed nodes that should be used for discovering other members
18 + * of the cluster.
19 + * @return set of seed controller nodes
20 + */
21 + Set<ControllerNode> seedNodes();
22 +
23 + /**
24 + * Forms cluster configuration based on the specified set of node
25 + * information. Assumes subsequent restart for the new configuration to
26 + * take hold.
27 + *
28 + * @param nodes set of nodes that form the cluster
29 + * @param ipPrefix IP address prefix, e.g. 10.0.1.*
30 + */
31 + void formCluster(Set<ControllerNode> nodes, String ipPrefix);
32 +}
...\ No newline at end of file ...\ No newline at end of file
...@@ -65,16 +65,6 @@ public interface ClusterStore extends Store<ClusterEvent, ClusterStoreDelegate> ...@@ -65,16 +65,6 @@ public interface ClusterStore extends Store<ClusterEvent, ClusterStoreDelegate>
65 DateTime getLastUpdated(NodeId nodeId); 65 DateTime getLastUpdated(NodeId nodeId);
66 66
67 /** 67 /**
68 - * Forms cluster configuration based on the specified set of node
69 - * information. Assumes subsequent restart for the new configuration to
70 - * take hold.
71 - *
72 - * @param nodes set of nodes that form the cluster
73 - * @param ipPrefix IP address prefix, e.g. 10.0.1.*
74 - */
75 - void formCluster(Set<ControllerNode> nodes, String ipPrefix);
76 -
77 - /**
78 * Adds a new controller node to the cluster. 68 * Adds a new controller node to the cluster.
79 * 69 *
80 * @param nodeId controller node identifier 70 * @param nodeId controller node identifier
......
...@@ -25,6 +25,7 @@ import org.apache.karaf.system.SystemService; ...@@ -25,6 +25,7 @@ import org.apache.karaf.system.SystemService;
25 import org.joda.time.DateTime; 25 import org.joda.time.DateTime;
26 import org.onlab.packet.IpAddress; 26 import org.onlab.packet.IpAddress;
27 import org.onosproject.cluster.ClusterAdminService; 27 import org.onosproject.cluster.ClusterAdminService;
28 +import org.onosproject.cluster.ClusterDefinitionService;
28 import org.onosproject.cluster.ClusterEvent; 29 import org.onosproject.cluster.ClusterEvent;
29 import org.onosproject.cluster.ClusterEventListener; 30 import org.onosproject.cluster.ClusterEventListener;
30 import org.onosproject.cluster.ClusterService; 31 import org.onosproject.cluster.ClusterService;
...@@ -58,6 +59,9 @@ public class ClusterManager implements ClusterService, ClusterAdminService { ...@@ -58,6 +59,9 @@ public class ClusterManager implements ClusterService, ClusterAdminService {
58 listenerRegistry = new ListenerRegistry<>(); 59 listenerRegistry = new ListenerRegistry<>();
59 60
60 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) 61 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
62 + protected ClusterDefinitionService clusterDefinitionService;
63 +
64 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
61 protected ClusterStore store; 65 protected ClusterStore store;
62 66
63 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) 67 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
...@@ -70,6 +74,8 @@ public class ClusterManager implements ClusterService, ClusterAdminService { ...@@ -70,6 +74,8 @@ public class ClusterManager implements ClusterService, ClusterAdminService {
70 public void activate() { 74 public void activate() {
71 store.setDelegate(delegate); 75 store.setDelegate(delegate);
72 eventDispatcher.addSink(ClusterEvent.class, listenerRegistry); 76 eventDispatcher.addSink(ClusterEvent.class, listenerRegistry);
77 + clusterDefinitionService.seedNodes()
78 + .forEach(node -> store.addNode(node.id(), node.ip(), node.tcpPort()));
73 log.info("Started"); 79 log.info("Started");
74 } 80 }
75 81
...@@ -113,7 +119,7 @@ public class ClusterManager implements ClusterService, ClusterAdminService { ...@@ -113,7 +119,7 @@ public class ClusterManager implements ClusterService, ClusterAdminService {
113 checkNotNull(nodes, "Nodes cannot be null"); 119 checkNotNull(nodes, "Nodes cannot be null");
114 checkArgument(!nodes.isEmpty(), "Nodes cannot be empty"); 120 checkArgument(!nodes.isEmpty(), "Nodes cannot be empty");
115 checkNotNull(ipPrefix, "IP prefix cannot be null"); 121 checkNotNull(ipPrefix, "IP prefix cannot be null");
116 - store.formCluster(nodes, ipPrefix); 122 + clusterDefinitionService.formCluster(nodes, ipPrefix);
117 try { 123 try {
118 log.warn("Shutting down container for cluster reconfiguration!"); 124 log.warn("Shutting down container for cluster reconfiguration!");
119 systemService.reboot("now", SystemService.Swipe.NONE); 125 systemService.reboot("now", SystemService.Swipe.NONE);
......
1 +package org.onosproject.store.cluster.impl;
2 +
3 +import static com.hazelcast.util.AddressUtil.matchInterface;
4 +import static java.net.NetworkInterface.getNetworkInterfaces;
5 +import static java.util.Collections.list;
6 +import static org.onosproject.cluster.DefaultControllerNode.DEFAULT_PORT;
7 +import static org.onosproject.store.consistent.impl.DatabaseManager.PARTITION_DEFINITION_FILE;
8 +import static org.slf4j.LoggerFactory.getLogger;
9 +
10 +import java.io.File;
11 +import java.io.IOException;
12 +import java.net.InetAddress;
13 +import java.net.NetworkInterface;
14 +import java.net.SocketException;
15 +import java.util.Enumeration;
16 +import java.util.Set;
17 +import java.util.stream.Collectors;
18 +
19 +import org.apache.felix.scr.annotations.Activate;
20 +import org.apache.felix.scr.annotations.Component;
21 +import org.apache.felix.scr.annotations.Deactivate;
22 +import org.apache.felix.scr.annotations.Service;
23 +import org.onlab.packet.IpAddress;
24 +import org.onosproject.cluster.ClusterDefinitionService;
25 +import org.onosproject.cluster.ControllerNode;
26 +import org.onosproject.cluster.DefaultControllerNode;
27 +import org.onosproject.cluster.NodeId;
28 +import org.onosproject.store.consistent.impl.DatabaseDefinition;
29 +import org.onosproject.store.consistent.impl.DatabaseDefinitionStore;
30 +import org.slf4j.Logger;
31 +
32 +import com.google.common.collect.ImmutableSet;
33 +import com.google.common.collect.Sets;
34 +import com.hazelcast.util.AddressUtil;
35 +
36 +/**
37 + * Implementation of ClusterDefinitionService.
38 + */
39 +@Component(immediate = true)
40 +@Service
41 +public class ClusterDefinitionManager implements ClusterDefinitionService {
42 +
43 + public static final String CLUSTER_DEFINITION_FILE = "../config/cluster.json";
44 + private static final String ONOS_NIC = "ONOS_NIC";
45 + private static final Logger log = getLogger(ClusterDefinitionManager.class);
46 + private ControllerNode localNode;
47 + private Set<ControllerNode> seedNodes;
48 +
49 + @Activate
50 + public void activate() {
51 + File clusterDefinitionFile = new File(CLUSTER_DEFINITION_FILE);
52 + ClusterDefinitionStore clusterDefinitionStore =
53 + new ClusterDefinitionStore(clusterDefinitionFile.getPath());
54 +
55 + if (!clusterDefinitionFile.exists()) {
56 + createDefaultClusterDefinition(clusterDefinitionStore);
57 + }
58 +
59 + try {
60 + ClusterDefinition clusterDefinition = clusterDefinitionStore.read();
61 + establishSelfIdentity(clusterDefinition);
62 + seedNodes = ImmutableSet
63 + .copyOf(clusterDefinition.getNodes())
64 + .stream()
65 + .filter(n -> !localNode.id().equals(new NodeId(n.getId())))
66 + .map(n -> new DefaultControllerNode(new NodeId(n.getId()),
67 + IpAddress.valueOf(n.getIp()),
68 + n.getTcpPort()))
69 + .collect(Collectors.toSet());
70 + } catch (IOException e) {
71 + throw new IllegalStateException("Failed to read cluster definition.", e);
72 + }
73 +
74 + log.info("Started");
75 + }
76 +
77 + @Deactivate
78 + public void deactivate() {
79 + log.info("Stopped");
80 + }
81 +
82 + @Override
83 + public ControllerNode localNode() {
84 + return localNode;
85 + }
86 +
87 + @Override
88 + public Set<ControllerNode> seedNodes() {
89 + return seedNodes;
90 + }
91 +
92 + @Override
93 + public void formCluster(Set<ControllerNode> nodes, String ipPrefix) {
94 + try {
95 + Set<NodeInfo> infos = Sets.newHashSet();
96 + nodes.forEach(n -> infos.add(NodeInfo.from(n.id().toString(),
97 + n.ip().toString(),
98 + n.tcpPort())));
99 +
100 + ClusterDefinition cdef = ClusterDefinition.from(infos, ipPrefix);
101 + new ClusterDefinitionStore(CLUSTER_DEFINITION_FILE).write(cdef);
102 +
103 + DatabaseDefinition ddef = DatabaseDefinition.from(infos);
104 + new DatabaseDefinitionStore(PARTITION_DEFINITION_FILE).write(ddef);
105 + } catch (IOException e) {
106 + log.error("Unable to form cluster", e);
107 + }
108 + }
109 +
110 + private IpAddress findLocalIp(ClusterDefinition clusterDefinition) throws SocketException {
111 + Enumeration<NetworkInterface> interfaces =
112 + NetworkInterface.getNetworkInterfaces();
113 + while (interfaces.hasMoreElements()) {
114 + NetworkInterface iface = interfaces.nextElement();
115 + Enumeration<InetAddress> inetAddresses = iface.getInetAddresses();
116 + while (inetAddresses.hasMoreElements()) {
117 + IpAddress ip = IpAddress.valueOf(inetAddresses.nextElement());
118 + if (AddressUtil.matchInterface(ip.toString(), clusterDefinition.getIpPrefix())) {
119 + return ip;
120 + }
121 + }
122 + }
123 + throw new IllegalStateException("Unable to determine local ip");
124 + }
125 +
126 + private void establishSelfIdentity(ClusterDefinition clusterDefinition) {
127 + try {
128 + IpAddress ip = findLocalIp(clusterDefinition);
129 + localNode = new DefaultControllerNode(new NodeId(ip.toString()), ip);
130 + } catch (SocketException e) {
131 + throw new IllegalStateException("Cannot determine local IP", e);
132 + }
133 + }
134 +
135 + private void createDefaultClusterDefinition(ClusterDefinitionStore store) {
136 + // Assumes IPv4 is returned.
137 + String ip = getSiteLocalAddress();
138 + String ipPrefix = ip.replaceFirst("\\.[0-9]*$", ".*");
139 + NodeInfo node = NodeInfo.from(ip, ip, DEFAULT_PORT);
140 + try {
141 + store.write(ClusterDefinition.from(ImmutableSet.of(node), ipPrefix));
142 + } catch (IOException e) {
143 + log.warn("Unable to write default cluster definition", e);
144 + }
145 + }
146 +
147 + /**
148 + * Returns the address that matches the IP prefix given in ONOS_NIC
149 + * environment variable if one was specified, or the first site local
150 + * address if one can be found or the loopback address otherwise.
151 + *
152 + * @return site-local address in string form
153 + */
154 + public static String getSiteLocalAddress() {
155 + try {
156 + String ipPrefix = System.getenv(ONOS_NIC);
157 + for (NetworkInterface nif : list(getNetworkInterfaces())) {
158 + for (InetAddress address : list(nif.getInetAddresses())) {
159 + IpAddress ip = IpAddress.valueOf(address);
160 + if (ipPrefix == null && address.isSiteLocalAddress() ||
161 + ipPrefix != null && matchInterface(ip.toString(), ipPrefix)) {
162 + return ip.toString();
163 + }
164 + }
165 + }
166 + } catch (SocketException e) {
167 + log.error("Unable to get network interfaces", e);
168 + }
169 +
170 + return IpAddress.valueOf(InetAddress.getLoopbackAddress()).toString();
171 + }
172 +}
...@@ -130,11 +130,6 @@ public class HazelcastClusterStore ...@@ -130,11 +130,6 @@ public class HazelcastClusterStore
130 } 130 }
131 131
132 @Override 132 @Override
133 - public void formCluster(Set<ControllerNode> nodes, String ipPrefix) {
134 - throw new UnsupportedOperationException("formCluster not implemented");
135 - }
136 -
137 - @Override
138 public ControllerNode addNode(NodeId nodeId, IpAddress ip, int tcpPort) { 133 public ControllerNode addNode(NodeId nodeId, IpAddress ip, int tcpPort) {
139 return addNode(new DefaultControllerNode(nodeId, ip, tcpPort)); 134 return addNode(new DefaultControllerNode(nodeId, ip, tcpPort));
140 } 135 }
......
...@@ -21,8 +21,6 @@ import org.apache.felix.scr.annotations.Deactivate; ...@@ -21,8 +21,6 @@ import org.apache.felix.scr.annotations.Deactivate;
21 import org.apache.felix.scr.annotations.Reference; 21 import org.apache.felix.scr.annotations.Reference;
22 import org.apache.felix.scr.annotations.ReferenceCardinality; 22 import org.apache.felix.scr.annotations.ReferenceCardinality;
23 import org.apache.felix.scr.annotations.Service; 23 import org.apache.felix.scr.annotations.Service;
24 -import org.onlab.netty.NettyMessagingManager;
25 -import org.onlab.nio.service.IOLoopMessagingManager;
26 import org.onosproject.cluster.ClusterService; 24 import org.onosproject.cluster.ClusterService;
27 import org.onosproject.cluster.ControllerNode; 25 import org.onosproject.cluster.ControllerNode;
28 import org.onosproject.cluster.NodeId; 26 import org.onosproject.cluster.NodeId;
...@@ -60,47 +58,16 @@ public class ClusterCommunicationManager ...@@ -60,47 +58,16 @@ public class ClusterCommunicationManager
60 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) 58 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
61 private ClusterService clusterService; 59 private ClusterService clusterService;
62 60
63 - // TODO: This probably should not be a OSGi service. 61 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
64 - private MessagingService messagingService; 62 + protected MessagingService messagingService;
65 -
66 - private final boolean useNetty = true;
67 63
68 @Activate 64 @Activate
69 public void activate() { 65 public void activate() {
70 - ControllerNode localNode = clusterService.getLocalNode(); 66 + log.info("Started");
71 - if (useNetty) {
72 - NettyMessagingManager netty = new NettyMessagingManager(localNode.ip(), localNode.tcpPort());
73 - try {
74 - netty.activate();
75 - messagingService = netty;
76 - } catch (Exception e) {
77 - log.error("NettyMessagingService#activate", e);
78 - }
79 - } else {
80 - IOLoopMessagingManager ioLoop = new IOLoopMessagingManager(localNode.ip(), localNode.tcpPort());
81 - try {
82 - ioLoop.activate();
83 - messagingService = ioLoop;
84 - } catch (Exception e) {
85 - log.error("IOLoopMessagingService#activate", e);
86 - }
87 - }
88 - log.info("Started on {}:{}", localNode.ip(), localNode.tcpPort());
89 } 67 }
90 68
91 @Deactivate 69 @Deactivate
92 public void deactivate() { 70 public void deactivate() {
93 - // TODO: cleanup messageingService if needed.
94 - // FIXME: workaround until it becomes a service.
95 - try {
96 - if (useNetty) {
97 - ((NettyMessagingManager) messagingService).deactivate();
98 - } else {
99 - ((IOLoopMessagingManager) messagingService).deactivate();
100 - }
101 - } catch (Exception e) {
102 - log.error("MessagingService#deactivate", e);
103 - }
104 log.info("Stopped"); 71 log.info("Stopped");
105 } 72 }
106 73
......
1 +package org.onosproject.store.cluster.messaging.impl;
2 +
3 +import org.apache.felix.scr.annotations.Activate;
4 +import org.apache.felix.scr.annotations.Component;
5 +import org.apache.felix.scr.annotations.Deactivate;
6 +import org.apache.felix.scr.annotations.Reference;
7 +import org.apache.felix.scr.annotations.ReferenceCardinality;
8 +import org.apache.felix.scr.annotations.Service;
9 +import org.onlab.nio.service.IOLoopMessaging;
10 +import org.onosproject.cluster.ClusterDefinitionService;
11 +import org.onosproject.cluster.ControllerNode;
12 +import org.onosproject.store.cluster.messaging.Endpoint;
13 +import org.slf4j.Logger;
14 +import org.slf4j.LoggerFactory;
15 +
16 +/**
17 + * IOLoop based MessagingService.
18 + */
19 +@Component(immediate = true, enabled = false)
20 +@Service
21 +public class IOLoopMessagingManager extends IOLoopMessaging {
22 +
23 + private final Logger log = LoggerFactory.getLogger(getClass());
24 +
25 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
26 + protected ClusterDefinitionService clusterDefinitionService;
27 +
28 + @Activate
29 + public void activate() throws Exception {
30 + ControllerNode localNode = clusterDefinitionService.localNode();
31 + super.start(new Endpoint(localNode.ip(), localNode.tcpPort()));
32 + log.info("Started");
33 + }
34 +
35 + @Deactivate
36 + public void deactivate() throws Exception {
37 + super.stop();
38 + log.info("Stopped");
39 + }
40 +}
1 +package org.onosproject.store.cluster.messaging.impl;
2 +
3 +import org.apache.felix.scr.annotations.Activate;
4 +import org.apache.felix.scr.annotations.Component;
5 +import org.apache.felix.scr.annotations.Deactivate;
6 +import org.apache.felix.scr.annotations.Reference;
7 +import org.apache.felix.scr.annotations.ReferenceCardinality;
8 +import org.apache.felix.scr.annotations.Service;
9 +import org.onlab.netty.NettyMessaging;
10 +import org.onosproject.cluster.ClusterDefinitionService;
11 +import org.onosproject.cluster.ControllerNode;
12 +import org.onosproject.store.cluster.messaging.Endpoint;
13 +import org.slf4j.Logger;
14 +import org.slf4j.LoggerFactory;
15 +
16 +/**
17 + * Netty based MessagingService.
18 + */
19 +@Component(immediate = true, enabled = true)
20 +@Service
21 +public class NettyMessagingManager extends NettyMessaging {
22 +
23 + private final Logger log = LoggerFactory.getLogger(getClass());
24 +
25 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
26 + protected ClusterDefinitionService clusterDefinitionService;
27 +
28 + @Activate
29 + public void activate() throws Exception {
30 + ControllerNode localNode = clusterDefinitionService.localNode();
31 + super.start(new Endpoint(localNode.ip(), localNode.tcpPort()));
32 + log.info("Started");
33 + }
34 +
35 + @Deactivate
36 + public void deactivate() throws Exception {
37 + super.stop();
38 + log.info("Stopped");
39 + }
40 +}
...\ No newline at end of file ...\ No newline at end of file
...@@ -45,7 +45,7 @@ import org.apache.felix.scr.annotations.ReferenceCardinality; ...@@ -45,7 +45,7 @@ import org.apache.felix.scr.annotations.ReferenceCardinality;
45 import org.apache.felix.scr.annotations.Service; 45 import org.apache.felix.scr.annotations.Service;
46 import org.onosproject.cluster.ClusterService; 46 import org.onosproject.cluster.ClusterService;
47 import org.onosproject.core.IdGenerator; 47 import org.onosproject.core.IdGenerator;
48 -import org.onosproject.store.cluster.impl.DistributedClusterStore; 48 +import org.onosproject.store.cluster.impl.ClusterDefinitionManager;
49 import org.onosproject.store.cluster.impl.NodeInfo; 49 import org.onosproject.store.cluster.impl.NodeInfo;
50 import org.onosproject.store.cluster.messaging.ClusterCommunicationService; 50 import org.onosproject.store.cluster.messaging.ClusterCommunicationService;
51 import org.onosproject.store.ecmap.EventuallyConsistentMapBuilderImpl; 51 import org.onosproject.store.ecmap.EventuallyConsistentMapBuilderImpl;
...@@ -193,7 +193,7 @@ public class DatabaseManager implements StorageService, StorageAdminService { ...@@ -193,7 +193,7 @@ public class DatabaseManager implements StorageService, StorageAdminService {
193 193
194 private void createDefaultDatabaseDefinition(DatabaseDefinitionStore store) { 194 private void createDefaultDatabaseDefinition(DatabaseDefinitionStore store) {
195 // Assumes IPv4 is returned. 195 // Assumes IPv4 is returned.
196 - String ip = DistributedClusterStore.getSiteLocalAddress(); 196 + String ip = ClusterDefinitionManager.getSiteLocalAddress();
197 NodeInfo node = NodeInfo.from(ip, ip, COPYCAT_TCP_PORT); 197 NodeInfo node = NodeInfo.from(ip, ip, COPYCAT_TCP_PORT);
198 try { 198 try {
199 store.write(DatabaseDefinition.from(ImmutableSet.of(node))); 199 store.write(DatabaseDefinition.from(ImmutableSet.of(node)));
......
...@@ -26,7 +26,7 @@ import org.apache.felix.scr.annotations.Activate; ...@@ -26,7 +26,7 @@ import org.apache.felix.scr.annotations.Activate;
26 import org.apache.felix.scr.annotations.Component; 26 import org.apache.felix.scr.annotations.Component;
27 import org.apache.felix.scr.annotations.Deactivate; 27 import org.apache.felix.scr.annotations.Deactivate;
28 import org.apache.felix.scr.annotations.Service; 28 import org.apache.felix.scr.annotations.Service;
29 -import org.onosproject.store.cluster.impl.DistributedClusterStore; 29 +import org.onosproject.store.cluster.impl.ClusterDefinitionManager;
30 import org.slf4j.Logger; 30 import org.slf4j.Logger;
31 import org.slf4j.LoggerFactory; 31 import org.slf4j.LoggerFactory;
32 32
...@@ -66,7 +66,7 @@ public class StoreManager implements StoreService { ...@@ -66,7 +66,7 @@ public class StoreManager implements StoreService {
66 } 66 }
67 67
68 private void createDefaultHazelcastFile(File hazelcastFile) { 68 private void createDefaultHazelcastFile(File hazelcastFile) {
69 - String ip = DistributedClusterStore.getSiteLocalAddress(); 69 + String ip = ClusterDefinitionManager.getSiteLocalAddress();
70 String ipPrefix = ip.replaceFirst("\\.[0-9]*$", ".*"); 70 String ipPrefix = ip.replaceFirst("\\.[0-9]*$", ".*");
71 InputStream his = getClass().getResourceAsStream("/hazelcast.xml"); 71 InputStream his = getClass().getResourceAsStream("/hazelcast.xml");
72 try { 72 try {
......
...@@ -140,7 +140,7 @@ public class ConsistentDeviceMastershipStore ...@@ -140,7 +140,7 @@ public class ConsistentDeviceMastershipStore
140 localNodeId = clusterService.getLocalNode().id(); 140 localNodeId = clusterService.getLocalNode().id();
141 leadershipService.addListener(leadershipEventListener); 141 leadershipService.addListener(leadershipEventListener);
142 142
143 - log.info("Started."); 143 + log.info("Started");
144 } 144 }
145 145
146 @Deactivate 146 @Deactivate
...@@ -151,7 +151,7 @@ public class ConsistentDeviceMastershipStore ...@@ -151,7 +151,7 @@ public class ConsistentDeviceMastershipStore
151 transferExecutor.shutdown(); 151 transferExecutor.shutdown();
152 leadershipService.removeListener(leadershipEventListener); 152 leadershipService.removeListener(leadershipEventListener);
153 153
154 - log.info("Stoppped."); 154 + log.info("Stopped");
155 } 155 }
156 156
157 @Override 157 @Override
......
...@@ -22,7 +22,6 @@ import org.junit.Test; ...@@ -22,7 +22,6 @@ import org.junit.Test;
22 import org.onosproject.cluster.DefaultControllerNode; 22 import org.onosproject.cluster.DefaultControllerNode;
23 import org.onosproject.cluster.NodeId; 23 import org.onosproject.cluster.NodeId;
24 import org.onosproject.store.cluster.impl.ClusterNodesDelegate; 24 import org.onosproject.store.cluster.impl.ClusterNodesDelegate;
25 -import org.onlab.netty.NettyMessagingManager;
26 import org.onlab.packet.IpAddress; 25 import org.onlab.packet.IpAddress;
27 26
28 import java.util.concurrent.CountDownLatch; 27 import java.util.concurrent.CountDownLatch;
......
...@@ -93,11 +93,6 @@ public class SimpleClusterStore ...@@ -93,11 +93,6 @@ public class SimpleClusterStore
93 } 93 }
94 94
95 @Override 95 @Override
96 - public void formCluster(Set<ControllerNode> nodes, String ipPrefix) {
97 -
98 - }
99 -
100 - @Override
101 public ControllerNode addNode(NodeId nodeId, IpAddress ip, int tcpPort) { 96 public ControllerNode addNode(NodeId nodeId, IpAddress ip, int tcpPort) {
102 return null; 97 return null;
103 } 98 }
......
...@@ -80,5 +80,4 @@ ...@@ -80,5 +80,4 @@
80 <version>${netty4.version}</version> 80 <version>${netty4.version}</version>
81 </dependency> 81 </dependency>
82 </dependencies> 82 </dependencies>
83 -
84 </project> 83 </project>
......
...@@ -37,22 +37,20 @@ import io.netty.channel.socket.nio.NioServerSocketChannel; ...@@ -37,22 +37,20 @@ import io.netty.channel.socket.nio.NioServerSocketChannel;
37 import io.netty.channel.socket.nio.NioSocketChannel; 37 import io.netty.channel.socket.nio.NioSocketChannel;
38 38
39 import java.io.IOException; 39 import java.io.IOException;
40 -import java.net.InetAddress; 40 +import java.util.Map;
41 -import java.net.UnknownHostException;
42 import java.util.concurrent.CompletableFuture; 41 import java.util.concurrent.CompletableFuture;
43 import java.util.concurrent.ConcurrentHashMap; 42 import java.util.concurrent.ConcurrentHashMap;
44 -import java.util.concurrent.ConcurrentMap;
45 import java.util.concurrent.Executor; 43 import java.util.concurrent.Executor;
46 import java.util.concurrent.RejectedExecutionException; 44 import java.util.concurrent.RejectedExecutionException;
47 import java.util.concurrent.TimeUnit; 45 import java.util.concurrent.TimeUnit;
48 import java.util.concurrent.TimeoutException; 46 import java.util.concurrent.TimeoutException;
47 +import java.util.concurrent.atomic.AtomicBoolean;
49 import java.util.concurrent.atomic.AtomicLong; 48 import java.util.concurrent.atomic.AtomicLong;
50 import java.util.function.Consumer; 49 import java.util.function.Consumer;
51 import java.util.function.Function; 50 import java.util.function.Function;
52 51
53 import org.apache.commons.pool.KeyedPoolableObjectFactory; 52 import org.apache.commons.pool.KeyedPoolableObjectFactory;
54 import org.apache.commons.pool.impl.GenericKeyedObjectPool; 53 import org.apache.commons.pool.impl.GenericKeyedObjectPool;
55 -import org.onlab.packet.IpAddress;
56 import org.onosproject.store.cluster.messaging.Endpoint; 54 import org.onosproject.store.cluster.messaging.Endpoint;
57 import org.onosproject.store.cluster.messaging.MessagingService; 55 import org.onosproject.store.cluster.messaging.MessagingService;
58 import org.slf4j.Logger; 56 import org.slf4j.Logger;
...@@ -66,14 +64,15 @@ import com.google.common.cache.RemovalNotification; ...@@ -66,14 +64,15 @@ import com.google.common.cache.RemovalNotification;
66 /** 64 /**
67 * Implementation of MessagingService based on <a href="http://netty.io/">Netty</a> framework. 65 * Implementation of MessagingService based on <a href="http://netty.io/">Netty</a> framework.
68 */ 66 */
69 -public class NettyMessagingManager implements MessagingService { 67 +public class NettyMessaging implements MessagingService {
70 68
71 private final Logger log = LoggerFactory.getLogger(getClass()); 69 private final Logger log = LoggerFactory.getLogger(getClass());
72 70
73 private static final String REPLY_MESSAGE_TYPE = "NETTY_MESSAGING_REQUEST_REPLY"; 71 private static final String REPLY_MESSAGE_TYPE = "NETTY_MESSAGING_REQUEST_REPLY";
74 72
75 - private final Endpoint localEp; 73 + private Endpoint localEp;
76 - private final ConcurrentMap<String, Consumer<InternalMessage>> handlers = new ConcurrentHashMap<>(); 74 + private final AtomicBoolean started = new AtomicBoolean(false);
75 + private final Map<String, Consumer<InternalMessage>> handlers = new ConcurrentHashMap<>();
77 private final AtomicLong messageIdGenerator = new AtomicLong(0); 76 private final AtomicLong messageIdGenerator = new AtomicLong(0);
78 private final Cache<Long, CompletableFuture<byte[]>> responseFutures = CacheBuilder.newBuilder() 77 private final Cache<Long, CompletableFuture<byte[]>> responseFutures = CacheBuilder.newBuilder()
79 .expireAfterWrite(10, TimeUnit.SECONDS) 78 .expireAfterWrite(10, TimeUnit.SECONDS)
...@@ -104,7 +103,8 @@ public class NettyMessagingManager implements MessagingService { ...@@ -104,7 +103,8 @@ public class NettyMessagingManager implements MessagingService {
104 clientChannelClass = EpollSocketChannel.class; 103 clientChannelClass = EpollSocketChannel.class;
105 return; 104 return;
106 } catch (Throwable e) { 105 } catch (Throwable e) {
107 - log.warn("Failed to initialize native (epoll) transport. Reason: {}. Proceeding with nio.", e.getMessage()); 106 + log.debug("Failed to initialize native (epoll) transport. "
107 + + "Reason: {}. Proceeding with nio.", e.getMessage());
108 } 108 }
109 clientGroup = new NioEventLoopGroup(); 109 clientGroup = new NioEventLoopGroup();
110 serverGroup = new NioEventLoopGroup(); 110 serverGroup = new NioEventLoopGroup();
...@@ -112,43 +112,27 @@ public class NettyMessagingManager implements MessagingService { ...@@ -112,43 +112,27 @@ public class NettyMessagingManager implements MessagingService {
112 clientChannelClass = NioSocketChannel.class; 112 clientChannelClass = NioSocketChannel.class;
113 } 113 }
114 114
115 - public NettyMessagingManager(IpAddress ip, int port) { 115 + public void start(Endpoint localEp) throws Exception {
116 - localEp = new Endpoint(ip, port); 116 + if (started.get()) {
117 - } 117 + log.warn("Already running at local endpoint: {}", localEp);
118 - 118 + return;
119 - public NettyMessagingManager() {
120 - this(8080);
121 - }
122 -
123 - public NettyMessagingManager(int port) {
124 - try {
125 - localEp = new Endpoint(IpAddress.valueOf(InetAddress.getLocalHost()), port);
126 - } catch (UnknownHostException e) {
127 - // Cannot resolve the local host, something is very wrong. Bailing out.
128 - throw new IllegalStateException("Cannot resolve local host", e);
129 - }
130 } 119 }
131 - 120 + this.localEp = localEp;
132 - public void activate() throws InterruptedException {
133 channels.setLifo(false); 121 channels.setLifo(false);
134 channels.setTestOnBorrow(true); 122 channels.setTestOnBorrow(true);
135 channels.setTestOnReturn(true); 123 channels.setTestOnReturn(true);
136 initEventLoopGroup(); 124 initEventLoopGroup();
137 startAcceptingConnections(); 125 startAcceptingConnections();
126 + started.set(true);
138 } 127 }
139 128
140 - public void deactivate() throws Exception { 129 + public void stop() throws Exception {
130 + if (started.get()) {
141 channels.close(); 131 channels.close();
142 serverGroup.shutdownGracefully(); 132 serverGroup.shutdownGracefully();
143 clientGroup.shutdownGracefully(); 133 clientGroup.shutdownGracefully();
134 + started.set(false);
144 } 135 }
145 -
146 - /**
147 - * Returns the local endpoint for this instance.
148 - * @return local end point.
149 - */
150 - public Endpoint localEp() {
151 - return localEp;
152 } 136 }
153 137
154 @Override 138 @Override
...@@ -237,7 +221,13 @@ public class NettyMessagingManager implements MessagingService { ...@@ -237,7 +221,13 @@ public class NettyMessagingManager implements MessagingService {
237 .childOption(ChannelOption.SO_KEEPALIVE, true); 221 .childOption(ChannelOption.SO_KEEPALIVE, true);
238 222
239 // Bind and start to accept incoming connections. 223 // Bind and start to accept incoming connections.
240 - b.bind(localEp.port()).sync(); 224 + b.bind(localEp.port()).sync().addListener(future -> {
225 + if (future.isSuccess()) {
226 + log.info("{} accepting incoming connections on port {}", localEp.host(), localEp.port());
227 + } else {
228 + log.warn("{} failed to bind to port {}", localEp.host(), localEp.port(), future.cause());
229 + }
230 + });
241 } 231 }
242 232
243 private class OnosCommunicationChannelFactory 233 private class OnosCommunicationChannelFactory
......
1 -/*
2 - * Copyright 2014-2015 Open Networking Laboratory
3 - *
4 - * Licensed under the Apache License, Version 2.0 (the "License");
5 - * you may not use this file except in compliance with the License.
6 - * You may obtain a copy of the License at
7 - *
8 - * http://www.apache.org/licenses/LICENSE-2.0
9 - *
10 - * Unless required by applicable law or agreed to in writing, software
11 - * distributed under the License is distributed on an "AS IS" BASIS,
12 - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 - * See the License for the specific language governing permissions and
14 - * limitations under the License.
15 - */
16 -package org.onlab.netty;
17 -
18 -import static org.junit.Assert.assertArrayEquals;
19 -
20 -import java.net.InetAddress;
21 -import java.util.concurrent.CompletableFuture;
22 -import java.util.concurrent.TimeUnit;
23 -import java.util.function.Function;
24 -
25 -import org.apache.commons.lang3.RandomUtils;
26 -import org.junit.Ignore;
27 -import org.junit.Test;
28 -import org.onlab.packet.IpAddress;
29 -import org.onosproject.store.cluster.messaging.Endpoint;
30 -
31 -import com.google.common.util.concurrent.MoreExecutors;
32 -
33 -/**
34 - * Simple ping-pong test that exercises NettyMessagingService.
35 - */
36 -public class PingPongTest {
37 -
38 - @Ignore("Turning off fragile test")
39 - @Test
40 - public void testPingPong() throws Exception {
41 - NettyMessagingManager pinger = new NettyMessagingManager(8085);
42 - NettyMessagingManager ponger = new NettyMessagingManager(9086);
43 - try {
44 - pinger.activate();
45 - ponger.activate();
46 - ponger.registerHandler("echo", Function.identity(), MoreExecutors.directExecutor());
47 - byte[] payload = RandomUtils.nextBytes(100);
48 - CompletableFuture<byte[]> responseFuture =
49 - pinger.sendAndReceive(
50 - new Endpoint(IpAddress.valueOf(InetAddress.getLocalHost()), 9086), "echo", payload);
51 - assertArrayEquals(payload, responseFuture.get(10000, TimeUnit.MILLISECONDS));
52 - } finally {
53 - pinger.deactivate();
54 - ponger.deactivate();
55 - }
56 - }
57 -}
...@@ -55,5 +55,4 @@ ...@@ -55,5 +55,4 @@
55 <scope>test</scope> 55 <scope>test</scope>
56 </dependency> 56 </dependency>
57 </dependencies> 57 </dependencies>
58 -
59 </project> 58 </project>
......
...@@ -31,6 +31,7 @@ import java.util.concurrent.Executor; ...@@ -31,6 +31,7 @@ import java.util.concurrent.Executor;
31 import java.util.concurrent.ExecutorService; 31 import java.util.concurrent.ExecutorService;
32 import java.util.concurrent.Executors; 32 import java.util.concurrent.Executors;
33 import java.util.concurrent.TimeoutException; 33 import java.util.concurrent.TimeoutException;
34 +import java.util.concurrent.atomic.AtomicBoolean;
34 import java.util.concurrent.atomic.AtomicLong; 35 import java.util.concurrent.atomic.AtomicLong;
35 import java.util.function.Consumer; 36 import java.util.function.Consumer;
36 import java.util.function.Function; 37 import java.util.function.Function;
...@@ -39,7 +40,6 @@ import org.apache.commons.pool.KeyedPoolableObjectFactory; ...@@ -39,7 +40,6 @@ import org.apache.commons.pool.KeyedPoolableObjectFactory;
39 import org.apache.commons.pool.impl.GenericKeyedObjectPool; 40 import org.apache.commons.pool.impl.GenericKeyedObjectPool;
40 import org.onlab.nio.AcceptorLoop; 41 import org.onlab.nio.AcceptorLoop;
41 import org.onlab.nio.SelectorLoop; 42 import org.onlab.nio.SelectorLoop;
42 -import org.onlab.packet.IpAddress;
43 import org.onosproject.store.cluster.messaging.Endpoint; 43 import org.onosproject.store.cluster.messaging.Endpoint;
44 import org.onosproject.store.cluster.messaging.MessagingService; 44 import org.onosproject.store.cluster.messaging.MessagingService;
45 import org.slf4j.Logger; 45 import org.slf4j.Logger;
...@@ -54,13 +54,12 @@ import com.google.common.collect.Lists; ...@@ -54,13 +54,12 @@ import com.google.common.collect.Lists;
54 /** 54 /**
55 * MessagingService implementation based on IOLoop. 55 * MessagingService implementation based on IOLoop.
56 */ 56 */
57 -public class IOLoopMessagingManager implements MessagingService { 57 +public class IOLoopMessaging implements MessagingService {
58 58
59 private final Logger log = LoggerFactory.getLogger(getClass()); 59 private final Logger log = LoggerFactory.getLogger(getClass());
60 60
61 private static final String REPLY_MESSAGE_TYPE = "ONOS_REQUEST_REPLY"; 61 private static final String REPLY_MESSAGE_TYPE = "ONOS_REQUEST_REPLY";
62 62
63 - static final int PORT = 9876;
64 static final long TIMEOUT = 1000; 63 static final long TIMEOUT = 1000;
65 64
66 static final boolean SO_NO_DELAY = false; 65 static final boolean SO_NO_DELAY = false;
...@@ -79,7 +78,8 @@ public class IOLoopMessagingManager implements MessagingService { ...@@ -79,7 +78,8 @@ public class IOLoopMessagingManager implements MessagingService {
79 78
80 private int lastWorker = -1; 79 private int lastWorker = -1;
81 80
82 - private final Endpoint localEp; 81 + private final AtomicBoolean started = new AtomicBoolean(false);
82 + private Endpoint localEp;
83 83
84 private GenericKeyedObjectPool<Endpoint, DefaultMessageStream> streams = 84 private GenericKeyedObjectPool<Endpoint, DefaultMessageStream> streams =
85 new GenericKeyedObjectPool<>(new DefaultMessageStreamFactory()); 85 new GenericKeyedObjectPool<>(new DefaultMessageStreamFactory());
...@@ -97,34 +97,17 @@ public class IOLoopMessagingManager implements MessagingService { ...@@ -97,34 +97,17 @@ public class IOLoopMessagingManager implements MessagingService {
97 }) 97 })
98 .build(); 98 .build();
99 99
100 -
101 - public IOLoopMessagingManager(int port) {
102 - this(new Endpoint(IpAddress.valueOf("127.0.0.1"), port));
103 - }
104 -
105 - public IOLoopMessagingManager(IpAddress ip, int port) {
106 - this(new Endpoint(ip, port));
107 - }
108 -
109 - public IOLoopMessagingManager(Endpoint localEp) {
110 - this.localEp = localEp;
111 - }
112 -
113 - /**
114 - * Returns the local endpoint.
115 - *
116 - * @return local endpoint
117 - */
118 - public Endpoint localEp() {
119 - return localEp;
120 - }
121 -
122 /** 100 /**
123 * Activates IO Loops. 101 * Activates IO Loops.
124 * 102 *
125 * @throws IOException is activation fails 103 * @throws IOException is activation fails
126 */ 104 */
127 - public void activate() throws IOException { 105 + public void start(Endpoint localEp) throws IOException {
106 + if (started.get()) {
107 + log.warn("IOMessaging is already running at {}", localEp);
108 + return;
109 + }
110 + this.localEp = localEp;
128 streams.setLifo(false); 111 streams.setLifo(false);
129 this.acceptorLoop = new DefaultAcceptorLoop(new InetSocketAddress(localEp.host().toString(), localEp.port())); 112 this.acceptorLoop = new DefaultAcceptorLoop(new InetSocketAddress(localEp.host().toString(), localEp.port()));
130 113
...@@ -136,16 +119,20 @@ public class IOLoopMessagingManager implements MessagingService { ...@@ -136,16 +119,20 @@ public class IOLoopMessagingManager implements MessagingService {
136 acceptorThreadPool.execute(acceptorLoop); 119 acceptorThreadPool.execute(acceptorLoop);
137 ioLoops.forEach(loop -> loop.awaitStart(TIMEOUT)); 120 ioLoops.forEach(loop -> loop.awaitStart(TIMEOUT));
138 acceptorLoop.awaitStart(TIMEOUT); 121 acceptorLoop.awaitStart(TIMEOUT);
122 + started.set(true);
139 } 123 }
140 124
141 /** 125 /**
142 * Shuts down IO loops. 126 * Shuts down IO loops.
143 */ 127 */
144 - public void deactivate() { 128 + public void stop() {
129 + if (started.get()) {
145 ioLoops.forEach(SelectorLoop::shutdown); 130 ioLoops.forEach(SelectorLoop::shutdown);
146 acceptorLoop.shutdown(); 131 acceptorLoop.shutdown();
147 ioThreadPool.shutdown(); 132 ioThreadPool.shutdown();
148 acceptorThreadPool.shutdown(); 133 acceptorThreadPool.shutdown();
134 + started.set(false);
135 + }
149 } 136 }
150 137
151 138
......