홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Madan Jampani
620f70d0 1 parent f6c973a0

LeadershipStore updates: 

 - Now tracking leader and candidates for a topic using a single map.
 - Using term numbers that are incremented by one every time a new leader is elected.
 - Introduced a separate LeadershipStore to conform to the  manager-store pattern

Change-Id: I1d03a6c5e8ff0e68ef0c1e3a6c2d425c4856e470
Inline Side-by-side
Showing 23 changed files with 897 additions and 1021 deletions
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
16 package org.onosproject.cip; 16 package org.onosproject.cip;
17 17
18 import com.google.common.io.ByteStreams; 18 import com.google.common.io.ByteStreams;
19 +
19 import org.apache.felix.scr.annotations.Activate; 20 import org.apache.felix.scr.annotations.Activate;
20 import org.apache.felix.scr.annotations.Component; 21 import org.apache.felix.scr.annotations.Component;
21 import org.apache.felix.scr.annotations.Deactivate; 22 import org.apache.felix.scr.annotations.Deactivate;
...@@ -93,7 +94,7 @@ public class ClusterIpManager { ...@@ -93,7 +94,7 @@ public class ClusterIpManager {
93 cfgService.registerProperties(getClass()); 94 cfgService.registerProperties(getClass());
94 95
95 localId = clusterService.getLocalNode().id(); 96 localId = clusterService.getLocalNode().id();
96 - processLeadershipChange(leadershipService.getLeader(CLUSTER_IP)); 97 + processLeaderChange(leadershipService.getLeader(CLUSTER_IP));
97 98
98 leadershipService.addListener(listener); 99 leadershipService.addListener(listener);
99 leadershipService.runForLeadership(CLUSTER_IP); 100 leadershipService.runForLeadership(CLUSTER_IP);
...@@ -137,10 +138,7 @@ public class ClusterIpManager { ...@@ -137,10 +138,7 @@ public class ClusterIpManager {
137 } 138 }
138 } 139 }
139 140
140 - private synchronized void processLeadershipChange(NodeId newLeader) { 141 + private synchronized void processLeaderChange(NodeId newLeader) {
141 - if (newLeader == null) {
142 - return;
143 - }
144 boolean isLeader = Objects.equals(newLeader, localId); 142 boolean isLeader = Objects.equals(newLeader, localId);
145 log.info("Processing leadership change; wasLeader={}, isLeader={}", wasLeader, isLeader); 143 log.info("Processing leadership change; wasLeader={}, isLeader={}", wasLeader, isLeader);
146 if (!wasLeader && isLeader) { 144 if (!wasLeader && isLeader) {
...@@ -189,11 +187,15 @@ public class ClusterIpManager { ...@@ -189,11 +187,15 @@ public class ClusterIpManager {
189 187
190 // Listens for leadership changes. 188 // Listens for leadership changes.
191 private class InternalLeadershipListener implements LeadershipEventListener { 189 private class InternalLeadershipListener implements LeadershipEventListener {
190 +
192 @Override 191 @Override
193 - public void event(LeadershipEvent event) { 192 + public boolean isRelevant(LeadershipEvent event) {
194 - if (event.subject().topic().equals(CLUSTER_IP)) { 193 + return CLUSTER_IP.equals(event.subject().topic());
195 - processLeadershipChange(event.subject().leader());
196 } 194 }
195 +
196 + @Override
197 + public void event(LeadershipEvent event) {
198 + processLeaderChange(event.subject().leaderNodeId());
197 } 199 }
198 } 200 }
199 201
......
...@@ -19,6 +19,7 @@ package org.onosproject.mlb; ...@@ -19,6 +19,7 @@ package org.onosproject.mlb;
19 import com.google.common.util.concurrent.ListenableScheduledFuture; 19 import com.google.common.util.concurrent.ListenableScheduledFuture;
20 import com.google.common.util.concurrent.ListeningScheduledExecutorService; 20 import com.google.common.util.concurrent.ListeningScheduledExecutorService;
21 import com.google.common.util.concurrent.MoreExecutors; 21 import com.google.common.util.concurrent.MoreExecutors;
22 +
22 import org.apache.felix.scr.annotations.Activate; 23 import org.apache.felix.scr.annotations.Activate;
23 import org.apache.felix.scr.annotations.Component; 24 import org.apache.felix.scr.annotations.Component;
24 import org.apache.felix.scr.annotations.Deactivate; 25 import org.apache.felix.scr.annotations.Deactivate;
...@@ -105,10 +106,7 @@ public class MastershipLoadBalancer { ...@@ -105,10 +106,7 @@ public class MastershipLoadBalancer {
105 log.info("Stopped"); 106 log.info("Stopped");
106 } 107 }
107 108
108 - private synchronized void processLeadershipChange(NodeId newLeader) { 109 + private synchronized void processLeaderChange(NodeId newLeader) {
109 - if (newLeader == null) {
110 - return;
111 - }
112 boolean currLeader = newLeader.equals(localId); 110 boolean currLeader = newLeader.equals(localId);
113 if (isLeader.getAndSet(currLeader) != currLeader) { 111 if (isLeader.getAndSet(currLeader) != currLeader) {
114 if (currLeader) { 112 if (currLeader) {
...@@ -159,7 +157,7 @@ public class MastershipLoadBalancer { ...@@ -159,7 +157,7 @@ public class MastershipLoadBalancer {
159 157
160 @Override 158 @Override
161 public void event(LeadershipEvent event) { 159 public void event(LeadershipEvent event) {
162 - processLeadershipChange(event.subject().leader()); 160 + processLeaderChange(event.subject().leaderNodeId());
163 } 161 }
164 } 162 }
165 } 163 }
......
...@@ -26,6 +26,7 @@ import org.onosproject.cluster.ClusterService; ...@@ -26,6 +26,7 @@ import org.onosproject.cluster.ClusterService;
26 import org.onosproject.cluster.LeadershipEvent; 26 import org.onosproject.cluster.LeadershipEvent;
27 import org.onosproject.cluster.LeadershipEventListener; 27 import org.onosproject.cluster.LeadershipEventListener;
28 import org.onosproject.cluster.LeadershipService; 28 import org.onosproject.cluster.LeadershipService;
29 +import org.onosproject.cluster.NodeId;
29 import org.onosproject.core.ApplicationId; 30 import org.onosproject.core.ApplicationId;
30 import org.onosproject.core.CoreService; 31 import org.onosproject.core.CoreService;
31 import org.onosproject.net.intent.Intent; 32 import org.onosproject.net.intent.Intent;
...@@ -43,7 +44,6 @@ import java.util.LinkedList; ...@@ -43,7 +44,6 @@ import java.util.LinkedList;
43 import java.util.List; 44 import java.util.List;
44 import java.util.Map; 45 import java.util.Map;
45 import java.util.Map.Entry; 46 import java.util.Map.Entry;
46 -import java.util.Objects;
47 import java.util.concurrent.ConcurrentHashMap; 47 import java.util.concurrent.ConcurrentHashMap;
48 import java.util.concurrent.ExecutorService; 48 import java.util.concurrent.ExecutorService;
49 49
...@@ -74,6 +74,7 @@ public class IntentSynchronizer implements IntentSynchronizationService, ...@@ -74,6 +74,7 @@ public class IntentSynchronizer implements IntentSynchronizationService,
74 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) 74 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
75 protected IntentService intentService; 75 protected IntentService intentService;
76 76
77 + private NodeId localNodeId;
77 private ApplicationId appId; 78 private ApplicationId appId;
78 79
79 private final InternalLeadershipListener leadershipEventListener = 80 private final InternalLeadershipListener leadershipEventListener =
...@@ -89,7 +90,7 @@ public class IntentSynchronizer implements IntentSynchronizationService, ...@@ -89,7 +90,7 @@ public class IntentSynchronizer implements IntentSynchronizationService,
89 @Activate 90 @Activate
90 public void activate() { 91 public void activate() {
91 intentsSynchronizerExecutor = createExecutor(); 92 intentsSynchronizerExecutor = createExecutor();
92 - 93 + this.localNodeId = clusterService.getLocalNode().id();
93 this.appId = coreService.registerApplication(APP_NAME); 94 this.appId = coreService.registerApplication(APP_NAME);
94 95
95 leadershipService.addListener(leadershipEventListener); 96 leadershipService.addListener(leadershipEventListener);
...@@ -268,27 +269,22 @@ public class IntentSynchronizer implements IntentSynchronizationService, ...@@ -268,27 +269,22 @@ public class IntentSynchronizer implements IntentSynchronizationService,
268 private class InternalLeadershipListener implements LeadershipEventListener { 269 private class InternalLeadershipListener implements LeadershipEventListener {
269 270
270 @Override 271 @Override
271 - public void event(LeadershipEvent event) { 272 + public boolean isRelevant(LeadershipEvent event) {
272 - if (!event.subject().topic().equals(appId.name())) { 273 + return event.subject().topic().equals(appId.name());
273 - // Not our topic: ignore
274 - return;
275 - }
276 - if (!Objects.equals(event.subject().leader(),
277 - clusterService.getLocalNode().id())) {
278 - // The event is not about this instance: ignore
279 - return;
280 } 274 }
281 275
276 + @Override
277 + public void event(LeadershipEvent event) {
282 switch (event.type()) { 278 switch (event.type()) {
283 - case LEADER_ELECTED: 279 + case LEADER_CHANGED:
280 + case LEADER_AND_CANDIDATES_CHANGED:
281 + if (localNodeId.equals(event.subject().leaderNodeId())) {
284 log.info("IntentSynchronizer gained leadership"); 282 log.info("IntentSynchronizer gained leadership");
285 leaderChanged(true); 283 leaderChanged(true);
286 - break; 284 + } else {
287 - case LEADER_BOOTED: 285 + log.info("IntentSynchronizer leader changed. New leader is {}", event.subject().leaderNodeId());
288 - log.info("IntentSynchronizer lost leadership");
289 leaderChanged(false); 286 leaderChanged(false);
290 - break; 287 + }
291 - case LEADER_REELECTED:
292 default: 288 default:
293 break; 289 break;
294 } 290 }
......
...@@ -17,6 +17,7 @@ package org.onosproject.routing.impl; ...@@ -17,6 +17,7 @@ package org.onosproject.routing.impl;
17 17
18 import com.google.common.collect.Sets; 18 import com.google.common.collect.Sets;
19 import com.google.common.util.concurrent.MoreExecutors; 19 import com.google.common.util.concurrent.MoreExecutors;
20 +
20 import org.junit.Before; 21 import org.junit.Before;
21 import org.junit.Test; 22 import org.junit.Test;
22 import org.onlab.junit.TestUtils; 23 import org.onlab.junit.TestUtils;
...@@ -29,7 +30,11 @@ import org.onlab.packet.IpPrefix; ...@@ -29,7 +30,11 @@ import org.onlab.packet.IpPrefix;
29 import org.onlab.packet.MacAddress; 30 import org.onlab.packet.MacAddress;
30 import org.onlab.packet.VlanId; 31 import org.onlab.packet.VlanId;
31 import org.onosproject.TestApplicationId; 32 import org.onosproject.TestApplicationId;
33 +import org.onosproject.cluster.ClusterServiceAdapter;
34 +import org.onosproject.cluster.ControllerNode;
35 +import org.onosproject.cluster.DefaultControllerNode;
32 import org.onosproject.cluster.LeadershipServiceAdapter; 36 import org.onosproject.cluster.LeadershipServiceAdapter;
37 +import org.onosproject.cluster.NodeId;
33 import org.onosproject.core.ApplicationId; 38 import org.onosproject.core.ApplicationId;
34 import org.onosproject.core.CoreServiceAdapter; 39 import org.onosproject.core.CoreServiceAdapter;
35 import org.onosproject.incubator.net.intf.Interface; 40 import org.onosproject.incubator.net.intf.Interface;
...@@ -94,6 +99,9 @@ public class IntentSynchronizerTest extends AbstractIntentTest { ...@@ -94,6 +99,9 @@ public class IntentSynchronizerTest extends AbstractIntentTest {
94 private static final ApplicationId APPID = 99 private static final ApplicationId APPID =
95 TestApplicationId.create("intent-sync-test"); 100 TestApplicationId.create("intent-sync-test");
96 101
102 + private static final ControllerNode LOCAL_NODE =
103 + new DefaultControllerNode(new NodeId("foo"), IpAddress.valueOf("127.0.0.1"));
104 +
97 @Before 105 @Before
98 public void setUp() throws Exception { 106 public void setUp() throws Exception {
99 super.setUp(); 107 super.setUp();
...@@ -105,6 +113,7 @@ public class IntentSynchronizerTest extends AbstractIntentTest { ...@@ -105,6 +113,7 @@ public class IntentSynchronizerTest extends AbstractIntentTest {
105 intentSynchronizer = new TestIntentSynchronizer(); 113 intentSynchronizer = new TestIntentSynchronizer();
106 114
107 intentSynchronizer.coreService = new TestCoreService(); 115 intentSynchronizer.coreService = new TestCoreService();
116 + intentSynchronizer.clusterService = new TestClusterService();
108 intentSynchronizer.leadershipService = new TestLeadershipService(); 117 intentSynchronizer.leadershipService = new TestLeadershipService();
109 intentSynchronizer.intentService = intentService; 118 intentSynchronizer.intentService = intentService;
110 119
...@@ -441,6 +450,13 @@ public class IntentSynchronizerTest extends AbstractIntentTest { ...@@ -441,6 +450,13 @@ public class IntentSynchronizerTest extends AbstractIntentTest {
441 } 450 }
442 } 451 }
443 452
453 + private class TestClusterService extends ClusterServiceAdapter {
454 + @Override
455 + public ControllerNode getLocalNode() {
456 + return LOCAL_NODE;
457 + }
458 + }
459 +
444 private class TestLeadershipService extends LeadershipServiceAdapter { 460 private class TestLeadershipService extends LeadershipServiceAdapter {
445 461
446 } 462 }
......
...@@ -24,12 +24,11 @@ import org.apache.felix.scr.annotations.Reference; ...@@ -24,12 +24,11 @@ import org.apache.felix.scr.annotations.Reference;
24 import org.apache.felix.scr.annotations.ReferenceCardinality; 24 import org.apache.felix.scr.annotations.ReferenceCardinality;
25 import org.onosproject.cluster.ClusterService; 25 import org.onosproject.cluster.ClusterService;
26 import org.onosproject.core.CoreService; 26 import org.onosproject.core.CoreService;
27 -import org.onosproject.cluster.ControllerNode;
28 import org.onosproject.cluster.LeadershipEvent; 27 import org.onosproject.cluster.LeadershipEvent;
29 import org.onosproject.cluster.LeadershipEventListener; 28 import org.onosproject.cluster.LeadershipEventListener;
30 import org.onosproject.cluster.LeadershipService; 29 import org.onosproject.cluster.LeadershipService;
30 +import org.onosproject.cluster.NodeId;
31 import org.onosproject.core.ApplicationId; 31 import org.onosproject.core.ApplicationId;
32 -
33 import org.slf4j.Logger; 32 import org.slf4j.Logger;
34 33
35 34
...@@ -56,7 +55,7 @@ public class ElectionTest { ...@@ -56,7 +55,7 @@ public class ElectionTest {
56 private LeadershipEventListener leadershipEventListener = 55 private LeadershipEventListener leadershipEventListener =
57 new InnerLeadershipEventListener(); 56 new InnerLeadershipEventListener();
58 57
59 - private ControllerNode localControllerNode; 58 + private NodeId localNodeId;
60 59
61 60
62 @Activate 61 @Activate
...@@ -65,7 +64,7 @@ public class ElectionTest { ...@@ -65,7 +64,7 @@ public class ElectionTest {
65 64
66 appId = coreService.registerApplication(ELECTION_APP); 65 appId = coreService.registerApplication(ELECTION_APP);
67 66
68 - localControllerNode = clusterService.getLocalNode(); 67 + localNodeId = clusterService.getLocalNode().id();
69 68
70 leadershipService.addListener(leadershipEventListener); 69 leadershipService.addListener(leadershipEventListener);
71 } 70 }
...@@ -100,20 +99,10 @@ public class ElectionTest { ...@@ -100,20 +99,10 @@ public class ElectionTest {
100 log.debug("Leadership Event: time = {} type = {} event = {}", 99 log.debug("Leadership Event: time = {} type = {} event = {}",
101 event.time(), event.type(), event); 100 event.time(), event.type(), event);
102 101
103 - if (!event.subject().leader().equals(
104 - localControllerNode.id())) {
105 - return; // The event is not about this instance: ignore
106 - }
107 -
108 switch (event.type()) { 102 switch (event.type()) {
109 - case LEADER_ELECTED: 103 + case LEADER_CHANGED:
110 - log.info("Election-test app leader elected"); 104 + case LEADER_AND_CANDIDATES_CHANGED:
111 - break; 105 + log.info("Election-test app leader changed. New leadership: {}", event.subject());
112 - case LEADER_BOOTED:
113 - log.info("Election-test app lost election");
114 - break;
115 - case LEADER_REELECTED:
116 - log.debug("Election-test app was re-elected");
117 break; 106 break;
118 default: 107 default:
119 break; 108 break;
......
...@@ -40,7 +40,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; ...@@ -40,7 +40,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
40 description = "Finds the leader for particular topic.") 40 description = "Finds the leader for particular topic.")
41 public class LeaderCommand extends AbstractShellCommand { 41 public class LeaderCommand extends AbstractShellCommand {
42 42
43 - private static final String FMT = "%-30s | %-15s | %-6s | %-10s |"; 43 + private static final String FMT = "%-30s | %-15s | %-5s | %-10s |";
44 private static final String FMT_C = "%-30s | %-15s | %-19s |"; 44 private static final String FMT_C = "%-30s | %-15s | %-19s |";
45 private boolean allTopics; 45 private boolean allTopics;
46 private Pattern pattern; 46 private Pattern pattern;
...@@ -57,19 +57,8 @@ public class LeaderCommand extends AbstractShellCommand { ...@@ -57,19 +57,8 @@ public class LeaderCommand extends AbstractShellCommand {
57 /** 57 /**
58 * Compares leaders, sorting by toString() output. 58 * Compares leaders, sorting by toString() output.
59 */ 59 */
60 - private Comparator<Leadership> leadershipComparator = 60 + private Comparator<Leadership> leadershipComparator = (l1, l2) ->
61 - (e1, e2) -> { 61 + String.valueOf(l1.leaderNodeId()).compareTo(String.valueOf(l2.leaderNodeId()));
62 - if (e1.leader() == null && e2.leader() == null) {
63 - return 0;
64 - }
65 - if (e1.leader() == null) {
66 - return 1;
67 - }
68 - if (e2.leader() == null) {
69 - return -1;
70 - }
71 - return e1.leader().toString().compareTo(e2.leader().toString());
72 - };
73 62
74 /** 63 /**
75 * Displays text representing the leaders. 64 * Displays text representing the leaders.
...@@ -78,18 +67,19 @@ public class LeaderCommand extends AbstractShellCommand { ...@@ -78,18 +67,19 @@ public class LeaderCommand extends AbstractShellCommand {
78 */ 67 */
79 private void displayLeaders(Map<String, Leadership> leaderBoard) { 68 private void displayLeaders(Map<String, Leadership> leaderBoard) {
80 print("------------------------------------------------------------------------"); 69 print("------------------------------------------------------------------------");
81 - print(FMT, "Topic", "Leader", "Epoch", "Elected"); 70 + print(FMT, "Topic", "Leader", "Term", "Elected");
82 print("------------------------------------------------------------------------"); 71 print("------------------------------------------------------------------------");
83 72
84 leaderBoard.values() 73 leaderBoard.values()
85 .stream() 74 .stream()
86 .filter(l -> allTopics || pattern.matcher(l.topic()).matches()) 75 .filter(l -> allTopics || pattern.matcher(l.topic()).matches())
76 + .filter(l -> l.leader() != null)
87 .sorted(leadershipComparator) 77 .sorted(leadershipComparator)
88 .forEach(l -> print(FMT, 78 .forEach(l -> print(FMT,
89 l.topic(), 79 l.topic(),
90 - l.leader(), 80 + l.leaderNodeId(),
91 - l.epoch(), 81 + l.leader().term(),
92 - Tools.timeAgo(l.electedTime()))); 82 + Tools.timeAgo(l.leader().termStartTime())));
93 print("------------------------------------------------------------------------"); 83 print("------------------------------------------------------------------------");
94 } 84 }
95 85
...@@ -110,7 +100,7 @@ public class LeaderCommand extends AbstractShellCommand { ...@@ -110,7 +100,7 @@ public class LeaderCommand extends AbstractShellCommand {
110 Leadership l = leaderBoard.get(es.getKey()); 100 Leadership l = leaderBoard.get(es.getKey());
111 print(FMT_C, 101 print(FMT_C,
112 es.getKey(), 102 es.getKey(),
113 - l == null ? "null" : l.leader(), 103 + String.valueOf(l.leaderNodeId()),
114 // formatting hacks to get it into a table 104 // formatting hacks to get it into a table
115 list.get(0).toString()); 105 list.get(0).toString());
116 list.subList(1, list.size()).forEach(n -> print(FMT_C, " ", " ", n)); 106 list.subList(1, list.size()).forEach(n -> print(FMT_C, " ", " ", n));
...@@ -134,10 +124,10 @@ public class LeaderCommand extends AbstractShellCommand { ...@@ -134,10 +124,10 @@ public class LeaderCommand extends AbstractShellCommand {
134 result.add( 124 result.add(
135 mapper.createObjectNode() 125 mapper.createObjectNode()
136 .put("topic", l.topic()) 126 .put("topic", l.topic())
137 - .put("leader", l.leader().toString()) 127 + .put("leader", String.valueOf(l.leaderNodeId()))
138 .put("candidates", l.candidates().toString()) 128 .put("candidates", l.candidates().toString())
139 - .put("epoch", l.epoch()) 129 + .put("epoch", l.leader().term())
140 - .put("electedTime", Tools.timeAgo(l.electedTime())))); 130 + .put("epochStartTime", Tools.timeAgo(l.leader().termStartTime()))));
141 131
142 return result; 132 return result;
143 } 133 }
......
1 +/*
2 + * Copyright 2016 Open Networking Laboratory
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +package org.onosproject.cluster;
17 +
18 +import com.google.common.base.MoreObjects;
19 +import com.google.common.base.Objects;
20 +import static com.google.common.base.Preconditions.checkNotNull;
21 +import static com.google.common.base.Preconditions.checkArgument;
22 +
23 +/**
24 + * Topic leader.
25 + * <p>
26 + * Identified by the {@link NodeId node identifier} and a monotonically increasing term number.
27 + * The term number is incremented by one every time a new node is elected as leader.
28 + * Also available is the system clock time at the instant when this node was elected as leader.
29 + * Keep in mind though that as with any system clock based time stamps this particular information
30 + * susceptible to clock skew and should only be relied on for simple diagnostic purposes.
31 + */
32 +public class Leader {
33 + private final NodeId nodeId;
34 + private final long term;
35 + private final long termStartTime;
36 +
37 + public Leader(NodeId nodeId, long term, long termStartTime) {
38 + this.nodeId = checkNotNull(nodeId);
39 + checkArgument(term >= 0, "term must be non-negative");
40 + this.term = term;
41 + checkArgument(termStartTime >= 0, "termStartTime must be non-negative");
42 + this.termStartTime = termStartTime;
43 + }
44 +
45 + /**
46 + * Returns the identifier for of leader.
47 + * @return node identifier
48 + */
49 + public NodeId nodeId() {
50 + return nodeId;
51 + }
52 +
53 + /**
54 + * Returns the leader's term.
55 + * @return leader term
56 + */
57 + public long term() {
58 + return term;
59 + }
60 +
61 + /**
62 + * Returns the system time when the current leadership term started.
63 + * @return current leader term start time
64 + */
65 + public long termStartTime() {
66 + return termStartTime;
67 + }
68 +
69 + @Override
70 + public boolean equals(Object other) {
71 + if (this == other) {
72 + return true;
73 + }
74 + if (other != null && other instanceof Leader) {
75 + Leader that = (Leader) other;
76 + return Objects.equal(this.nodeId, that.nodeId) &&
77 + this.term == that.term &&
78 + this.termStartTime == that.termStartTime;
79 + }
80 + return false;
81 + }
82 +
83 + @Override
84 + public int hashCode() {
85 + return Objects.hashCode(nodeId, term, termStartTime);
86 + }
87 +
88 + @Override
89 + public String toString() {
90 + return MoreObjects.toStringHelper(getClass())
91 + .add("nodeId", nodeId)
92 + .add("term", term)
93 + .add("termStartTime", termStartTime)
94 + .toString();
95 + }
96 +}
...@@ -17,63 +17,31 @@ package org.onosproject.cluster; ...@@ -17,63 +17,31 @@ package org.onosproject.cluster;
17 17
18 import java.util.Objects; 18 import java.util.Objects;
19 import java.util.List; 19 import java.util.List;
20 -import java.util.Optional;
21 -
22 -import org.joda.time.DateTime;
23 20
24 import com.google.common.base.MoreObjects; 21 import com.google.common.base.MoreObjects;
25 import com.google.common.collect.ImmutableList; 22 import com.google.common.collect.ImmutableList;
26 23
27 /** 24 /**
28 - * Abstract leadership concept. The information carried by this construct 25 + * State of leadership for topic.
29 - * include the topic of contention, the {@link NodeId}s of Nodes that could 26 + * <p>
30 - * become leader for the topic, the epoch when the term for a given leader 27 + * Provided by this construct is the current {@link Leader leader} and the list of
31 - * began, and the system time when the term began. Note: 28 + * {@link NodeId nodeId}s currently registered as candidates for election for the topic.
32 - * <ul> 29 + * Keep in mind that only registered candidates can become leaders.
33 - * <li>The list of NodeIds may include the current leader at index 0, and the
34 - * rest in decreasing preference order.</li>
35 - * <li>The epoch is the logical age of a Leadership construct, and should be
36 - * used for comparing two Leaderships, but only of the same topic.</li>
37 - * <li>The leader may be null if its accuracy can't be guaranteed. This applies
38 - * to CANDIDATES_CHANGED events and candidate board contents.</li>
39 - * </ul>
40 */ 30 */
41 public class Leadership { 31 public class Leadership {
42 32
43 private final String topic; 33 private final String topic;
44 - private final Optional<NodeId> leader; 34 + private final Leader leader;
45 private final List<NodeId> candidates; 35 private final List<NodeId> candidates;
46 - private final long epoch;
47 - private final long electedTime;
48 36
49 - public Leadership(String topic, NodeId leader, long epoch, long electedTime) { 37 + public Leadership(String topic, Leader leader, List<NodeId> candidates) {
50 this.topic = topic; 38 this.topic = topic;
51 - this.leader = Optional.of(leader); 39 + this.leader = leader;
52 - this.candidates = ImmutableList.of(leader);
53 - this.epoch = epoch;
54 - this.electedTime = electedTime;
55 - }
56 -
57 - public Leadership(String topic, NodeId leader, List<NodeId> candidates,
58 - long epoch, long electedTime) {
59 - this.topic = topic;
60 - this.leader = Optional.of(leader);
61 this.candidates = ImmutableList.copyOf(candidates); 40 this.candidates = ImmutableList.copyOf(candidates);
62 - this.epoch = epoch;
63 - this.electedTime = electedTime;
64 - }
65 -
66 - public Leadership(String topic, List<NodeId> candidates,
67 - long epoch, long electedTime) {
68 - this.topic = topic;
69 - this.leader = Optional.empty();
70 - this.candidates = ImmutableList.copyOf(candidates);
71 - this.epoch = epoch;
72 - this.electedTime = electedTime;
73 } 41 }
74 42
75 /** 43 /**
76 - * The topic for which this leadership applies. 44 + * Returns the leadership topic.
77 * 45 *
78 * @return leadership topic. 46 * @return leadership topic.
79 */ 47 */
...@@ -82,57 +50,36 @@ public class Leadership { ...@@ -82,57 +50,36 @@ public class Leadership {
82 } 50 }
83 51
84 /** 52 /**
85 - * The nodeId of leader for this topic. 53 + * Returns the {@link NodeId nodeId} of the leader.
86 - *
87 - * @return leader node.
88 - */
89 - // This will return Optional<NodeId> in the future.
90 - public NodeId leader() {
91 - return leader.orElse(null);
92 - }
93 -
94 - /**
95 - * Returns an preference-ordered list of nodes that are in the leadership
96 - * race for this topic.
97 * 54 *
98 - * @return a list of NodeIds in priority-order, or an empty list. 55 + * @return leader node identifier; will be null if there is no leader
99 */ 56 */
100 - public List<NodeId> candidates() { 57 + public NodeId leaderNodeId() {
101 - return candidates; 58 + return leader == null ? null : leader.nodeId();
102 } 59 }
103 60
104 /** 61 /**
105 - * The epoch when the leadership was assumed. 62 + * Returns the leader for this topic.
106 - * <p>
107 - * Comparing epochs is only appropriate for leadership events for the same
108 - * topic. The system guarantees that for any given topic the epoch for a new
109 - * term is higher (not necessarily by 1) than the epoch for any previous
110 - * term.
111 * 63 *
112 - * @return leadership epoch 64 + * @return leader; will be null if there is no leader for topic
113 */ 65 */
114 - public long epoch() { 66 + public Leader leader() {
115 - return epoch; 67 + return leader;
116 } 68 }
117 69
118 /** 70 /**
119 - * The system time when the term started. 71 + * Returns an preference-ordered list of nodes that are in the leadership
120 - * <p> 72 + * race for this topic.
121 - * The elected time is initially set on the node coordinating
122 - * the leader election using its local system time. Due to possible
123 - * clock skew, relying on this value for determining event ordering
124 - * is discouraged. Epoch is more appropriate for determining
125 - * event ordering.
126 * 73 *
127 - * @return elected time. 74 + * @return a list of NodeIds in priority-order, or an empty list.
128 */ 75 */
129 - public long electedTime() { 76 + public List<NodeId> candidates() {
130 - return electedTime; 77 + return candidates;
131 } 78 }
132 79
133 @Override 80 @Override
134 public int hashCode() { 81 public int hashCode() {
135 - return Objects.hash(topic, leader, candidates, epoch, electedTime); 82 + return Objects.hash(topic, leader, candidates);
136 } 83 }
137 84
138 @Override 85 @Override
...@@ -144,9 +91,7 @@ public class Leadership { ...@@ -144,9 +91,7 @@ public class Leadership {
144 final Leadership other = (Leadership) obj; 91 final Leadership other = (Leadership) obj;
145 return Objects.equals(this.topic, other.topic) && 92 return Objects.equals(this.topic, other.topic) &&
146 Objects.equals(this.leader, other.leader) && 93 Objects.equals(this.leader, other.leader) &&
147 - Objects.equals(this.candidates, other.candidates) && 94 + Objects.equals(this.candidates, other.candidates);
148 - Objects.equals(this.epoch, other.epoch) &&
149 - Objects.equals(this.electedTime, other.electedTime);
150 } 95 }
151 return false; 96 return false;
152 } 97 }
...@@ -157,8 +102,6 @@ public class Leadership { ...@@ -157,8 +102,6 @@ public class Leadership {
157 .add("topic", topic) 102 .add("topic", topic)
158 .add("leader", leader) 103 .add("leader", leader)
159 .add("candidates", candidates) 104 .add("candidates", candidates)
160 - .add("epoch", epoch)
161 - .add("electedTime", new DateTime(electedTime))
162 .toString(); 105 .toString();
163 } 106 }
164 } 107 }
......
1 +/*
2 + * Copyright 2016 Open Networking Laboratory
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +package org.onosproject.cluster;
17 +
18 +/**
19 + * Interface for administratively manipulating leadership assignments.
20 + */
21 +public interface LeadershipAdminService {
22 +
23 + /**
24 + * Attempts to assign leadership for a topic to a specified node.
25 + * @param topic leadership topic
26 + * @param nodeId identifier of the node to be made leader
27 + * @return true is the transfer was successfully executed. This method returns {@code false}
28 + * if {@code nodeId} is not one of the candidates for for the topic.
29 + */
30 + boolean transferLeadership(String topic, NodeId nodeId);
31 +
32 + /**
33 + * Make a node to be the next leader by promoting it to top of candidate list.
34 + * @param topic leadership topic
35 + * @param nodeId identifier of node to be next leader
36 + * @return {@code true} if nodeId is now the top candidate. This method returns {@code false}
37 + * if {@code nodeId} is not one of the candidates for for the topic.
38 + */
39 + boolean promoteToTopOfCandidateList(String topic, NodeId nodeId);
40 +
41 + /**
42 + * Removes all active leadership registrations for a given node.
43 + * <p>
44 + * This method will also evict the node from leaderships that it currently owns.
45 + * @param nodeId node identifier
46 + */
47 + void unregister(NodeId nodeId);
48 +}
...@@ -27,33 +27,25 @@ import com.google.common.base.MoreObjects; ...@@ -27,33 +27,25 @@ import com.google.common.base.MoreObjects;
27 public class LeadershipEvent extends AbstractEvent<LeadershipEvent.Type, Leadership> { 27 public class LeadershipEvent extends AbstractEvent<LeadershipEvent.Type, Leadership> {
28 28
29 /** 29 /**
30 - * Type of leadership-related events. 30 + * Type of leadership events.
31 */ 31 */
32 public enum Type { 32 public enum Type {
33 /** 33 /**
34 - * Signifies that the leader has been elected. 34 + * Signifies a change in both the leader as well as change to the list of candidates. Keep in mind though that
35 - * The event subject is the new leader. 35 + * the first node entering the race will trigger this event as it will become a candidate and automatically get
36 - * This event does not guarantee accurate candidate information. 36 + * promoted to become leader.
37 */ 37 */
38 - LEADER_ELECTED, 38 + LEADER_AND_CANDIDATES_CHANGED,
39 39
40 /** 40 /**
41 - * Signifies that the leader has been re-elected. 41 + * Signifies that the leader for a topic has changed.
42 - * The event subject is the leader.
43 - * This event does not guarantee accurate candidate information.
44 */ 42 */
45 - LEADER_REELECTED, 43 + // TODO: We may not need this. We currently do not support a way for a current leader to step down
44 + // while still reamining a candidate
45 + LEADER_CHANGED,
46 46
47 /** 47 /**
48 - * Signifies that the leader has been booted and lost leadership. 48 + * Signifies a change in the list of candidates for a topic.
49 - * The event subject is the former leader.
50 - * This event does not guarantee accurate candidate information.
51 - */
52 - LEADER_BOOTED,
53 -
54 - /**
55 - * Signifies that the list of candidates for leadership for a topic has
56 - * changed. This event does not guarantee accurate leader information.
57 */ 49 */
58 CANDIDATES_CHANGED 50 CANDIDATES_CHANGED
59 } 51 }
......
...@@ -17,87 +17,73 @@ package org.onosproject.cluster; ...@@ -17,87 +17,73 @@ package org.onosproject.cluster;
17 17
18 import org.onosproject.event.ListenerService; 18 import org.onosproject.event.ListenerService;
19 19
20 +import com.google.common.base.Objects;
21 +import com.google.common.collect.ImmutableList;
22 +import com.google.common.collect.ImmutableMap;
23 +import com.google.common.collect.Maps;
24 +
20 import java.util.List; 25 import java.util.List;
21 import java.util.Map; 26 import java.util.Map;
22 import java.util.Set; 27 import java.util.Set;
23 -import java.util.concurrent.CompletableFuture;
24 28
25 /** 29 /**
26 * Service for leader election. 30 * Service for leader election.
31 + * <p>
27 * Leadership contests are organized around topics. A instance can join the 32 * Leadership contests are organized around topics. A instance can join the
28 * leadership race for a topic or withdraw from a race it has previously joined. 33 * leadership race for a topic or withdraw from a race it has previously joined.
34 + * <p>
29 * Listeners can be added to receive notifications asynchronously for various 35 * Listeners can be added to receive notifications asynchronously for various
30 * leadership contests. 36 * leadership contests.
37 + * <p>
38 + * When a node gets elected as a leader for a topic, all nodes receive notifications
39 + * indicating a change in leadership.
31 */ 40 */
32 public interface LeadershipService 41 public interface LeadershipService
33 extends ListenerService<LeadershipEvent, LeadershipEventListener> { 42 extends ListenerService<LeadershipEvent, LeadershipEventListener> {
34 43
35 /** 44 /**
36 - * Returns the current leader for the topic. 45 + * Returns the {@link NodeId node identifier} that is the current leader for a topic.
37 * 46 *
38 - * @param path topic 47 + * @param topic leadership topic
39 - * @return nodeId of the leader, null if so such topic exists. 48 + * @return node identifier of the current leader; {@code null} if there is no leader for the topic
40 */ 49 */
41 - NodeId getLeader(String path); 50 + default NodeId getLeader(String topic) {
51 + Leadership leadership = getLeadership(topic);
52 + return leadership == null ? null : leadership.leaderNodeId();
53 + }
42 54
43 /** 55 /**
44 - * Returns the current leadership info for the topic. 56 + * Returns the current {@link Leadership leadership} for a topic.
45 * 57 *
46 - * @param path topic 58 + * @param topic leadership topic
47 - * @return leadership info or null if so such topic exists. 59 + * @return leadership or {@code null} if no such topic exists
48 */ 60 */
49 - Leadership getLeadership(String path); 61 + Leadership getLeadership(String topic);
50 62
51 /** 63 /**
52 - * Returns the set of topics owned by the specified node. 64 + * Returns the set of topics owned by the specified {@link NodeId node}.
53 * 65 *
54 - * @param nodeId node Id. 66 + * @param nodeId node identifier.
55 * @return set of topics for which this node is the current leader. 67 * @return set of topics for which this node is the current leader.
56 */ 68 */
57 - Set<String> ownedTopics(NodeId nodeId); 69 + default Set<String> ownedTopics(NodeId nodeId) {
70 + return Maps.filterValues(getLeaderBoard(), v -> Objects.equal(nodeId, v.leaderNodeId())).keySet();
71 + }
58 72
59 /** 73 /**
60 - * Joins the leadership contest. 74 + * Enters a leadership contest.
61 * 75 *
62 - * @param path topic for which this controller node wishes to be a leader 76 + * @param topic leadership topic
63 * @return {@code Leadership} future 77 * @return {@code Leadership} future
64 */ 78 */
65 - CompletableFuture<Leadership> runForLeadership(String path); 79 + Leadership runForLeadership(String topic);
66 80
67 /** 81 /**
68 * Withdraws from a leadership contest. 82 * Withdraws from a leadership contest.
69 * 83 *
70 - * @param path topic for which this controller node no longer wishes to be a leader 84 + * @param topic leadership topic
71 - * @return future that is successfully completed when withdraw is done
72 - */
73 - CompletableFuture<Void> withdraw(String path);
74 -
75 - /**
76 - * If the local nodeId is the leader for specified topic, this method causes it to
77 - * step down temporarily from leadership.
78 - * <p>
79 - * The node will continue to be in contention for leadership and can
80 - * potentially become the leader again if and when it becomes the highest
81 - * priority candidate
82 - * <p>
83 - * If the local nodeId is not the leader, this method will make no changes and
84 - * simply return false.
85 - *
86 - * @param path topic for which this controller node should give up leadership
87 - * @return true if this node stepped down from leadership, false otherwise
88 - */
89 - boolean stepdown(String path);
90 -
91 - /**
92 - * Moves the specified nodeId to the top of the candidates list for the topic.
93 - * <p>
94 - * If the node is not a candidate for this topic, this method will be a noop.
95 - *
96 - * @param path leadership topic
97 - * @param nodeId nodeId to make the top candidate
98 - * @return true if nodeId is now the top candidate, false otherwise
99 */ 85 */
100 - boolean makeTopCandidate(String path, NodeId nodeId); 86 + void withdraw(String topic);
101 87
102 /** 88 /**
103 * Returns the current leader board. 89 * Returns the current leader board.
...@@ -107,18 +93,22 @@ public interface LeadershipService ...@@ -107,18 +93,22 @@ public interface LeadershipService
107 Map<String, Leadership> getLeaderBoard(); 93 Map<String, Leadership> getLeaderBoard();
108 94
109 /** 95 /**
110 - * Returns the candidates for all known topics. 96 + * Returns the candidate nodes for each topic.
111 * 97 *
112 * @return A mapping from topics to corresponding list of candidates. 98 * @return A mapping from topics to corresponding list of candidates.
113 */ 99 */
114 - Map<String, List<NodeId>> getCandidates(); 100 + default Map<String, List<NodeId>> getCandidates() {
101 + return ImmutableMap.copyOf(Maps.transformValues(getLeaderBoard(), v -> ImmutableList.copyOf(v.candidates())));
102 + }
115 103
116 /** 104 /**
117 - * Returns the candidates for a given topic. 105 + * Returns the candidate nodes for a given topic.
118 * 106 *
119 - * @param path topic 107 + * @param topic leadership topic
120 - * @return A lists of NodeIds, which may be empty. 108 + * @return A lists of {@link NodeId nodeIds}, which may be empty.
121 */ 109 */
122 - List<NodeId> getCandidates(String path); 110 + default List<NodeId> getCandidates(String topic) {
123 - 111 + Leadership leadership = getLeadership(topic);
112 + return leadership == null ? ImmutableList.of() : ImmutableList.copyOf(leadership.candidates());
113 + }
124 } 114 }
...\ No newline at end of file ...\ No newline at end of file
......
1 +/*
2 + * Copyright 2016 Open Networking Laboratory
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +package org.onosproject.cluster;
17 +
18 +import java.util.Map;
19 +import org.onosproject.store.Store;
20 +
21 +/**
22 + * Store interface for managing {@link LeadershipService} state.
23 + */
24 +public interface LeadershipStore extends Store<LeadershipEvent, LeadershipStoreDelegate> {
25 +
26 + /**
27 + * Adds registration for the local instance to be leader for topic.
28 + *
29 + * @param topic leadership topic
30 + * @return Updated leadership after operation is completed
31 + */
32 + Leadership addRegistration(String topic);
33 +
34 + /**
35 + * Unregisters the local instance from leadership contest for topic.
36 + *
37 + * @param topic leadership topic
38 + */
39 + void removeRegistration(String topic);
40 +
41 + /**
42 + * Unregisters an instance from all leadership contests.
43 + *
44 + * @param nodeId node identifier
45 + */
46 + void removeRegistration(NodeId nodeId);
47 +
48 + /**
49 + * Updates state so that given node is leader for a topic.
50 + *
51 + * @param topic leadership topic
52 + * @param toNodeId identifier of the desired leader
53 + * @return {@code true} if the transfer succeeded; {@code false} otherwise.
54 + * This method can return {@code false} if the node is not registered for the topic
55 + */
56 + boolean moveLeadership(String topic, NodeId toNodeId);
57 +
58 + /**
59 + * Attempts to make a node the top candidate.
60 + *
61 + * @param topic leadership topic
62 + * @param nodeId node identifier
63 + * @return {@code true} if the specified node is now the top candidate.
64 + * This method will return {@code false} if the node is not registered for the topic
65 + */
66 + boolean makeTopCandidate(String topic, NodeId nodeId);
67 +
68 + /**
69 + * Returns the current leadership for topic.
70 + *
71 + * @param topic leadership topic
72 + * @return current leadership
73 + */
74 + Leadership getLeadership(String topic);
75 +
76 + /**
77 + * Return current leadership for all topics.
78 + *
79 + * @return topic to leadership mapping
80 + */
81 + Map<String, Leadership> getLeaderships();
82 +}
...\ No newline at end of file ...\ No newline at end of file
1 +/*
2 + * Copyright 2015 Open Networking Laboratory
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +package org.onosproject.cluster;
17 +
18 +import org.onosproject.store.StoreDelegate;
19 +
20 +/**
21 + * {@link LeadershipStore} delegate abstraction.
22 + */
23 +public interface LeadershipStoreDelegate extends StoreDelegate<LeadershipEvent> {
24 +}
...@@ -15,6 +15,8 @@ ...@@ -15,6 +15,8 @@
15 */ 15 */
16 package org.onosproject.cluster; 16 package org.onosproject.cluster;
17 17
18 +import java.util.Arrays;
19 +
18 import org.junit.Test; 20 import org.junit.Test;
19 21
20 import com.google.common.testing.EqualsTester; 22 import com.google.common.testing.EqualsTester;
...@@ -28,10 +30,11 @@ import static org.junit.Assert.assertThat; ...@@ -28,10 +30,11 @@ import static org.junit.Assert.assertThat;
28 public class LeadershipEventTest { 30 public class LeadershipEventTest {
29 private final NodeId node1 = new NodeId("1"); 31 private final NodeId node1 = new NodeId("1");
30 private final NodeId node2 = new NodeId("2"); 32 private final NodeId node2 = new NodeId("2");
31 - private final Leadership lead1 = new Leadership("topic1", node1, 1L, 2L); 33 + private final Leadership lead1 = new Leadership("topic1", new Leader(node1, 1L, 2L), Arrays.asList(node1));
32 - private final Leadership lead2 = new Leadership("topic1", node2, 1L, 2L); 34 + private final Leadership lead2 = new Leadership("topic1", new Leader(node1, 1L, 2L), Arrays.asList(node1, node2));
35 + private final Leadership lead3 = new Leadership("topic1", new Leader(node2, 1L, 2L), Arrays.asList(node2));
33 private final LeadershipEvent event1 = 36 private final LeadershipEvent event1 =
34 - new LeadershipEvent(LeadershipEvent.Type.LEADER_ELECTED, lead1); 37 + new LeadershipEvent(LeadershipEvent.Type.LEADER_CHANGED, lead1);
35 private final long time = System.currentTimeMillis(); 38 private final long time = System.currentTimeMillis();
36 private final LeadershipEvent event2 = 39 private final LeadershipEvent event2 =
37 new LeadershipEvent(LeadershipEvent.Type.CANDIDATES_CHANGED, 40 new LeadershipEvent(LeadershipEvent.Type.CANDIDATES_CHANGED,
...@@ -40,11 +43,9 @@ public class LeadershipEventTest { ...@@ -40,11 +43,9 @@ public class LeadershipEventTest {
40 new LeadershipEvent(LeadershipEvent.Type.CANDIDATES_CHANGED, 43 new LeadershipEvent(LeadershipEvent.Type.CANDIDATES_CHANGED,
41 lead2, time); 44 lead2, time);
42 private final LeadershipEvent event3 = 45 private final LeadershipEvent event3 =
43 - new LeadershipEvent(LeadershipEvent.Type.LEADER_BOOTED, lead1); 46 + new LeadershipEvent(LeadershipEvent.Type.LEADER_CHANGED, lead2);
44 private final LeadershipEvent event4 = 47 private final LeadershipEvent event4 =
45 - new LeadershipEvent(LeadershipEvent.Type.LEADER_REELECTED, lead1); 48 + new LeadershipEvent(LeadershipEvent.Type.LEADER_AND_CANDIDATES_CHANGED, lead3);
46 - private final LeadershipEvent event5 =
47 - new LeadershipEvent(LeadershipEvent.Type.LEADER_REELECTED, lead2);
48 49
49 /** 50 /**
50 * Tests for proper operation of equals(), hashCode() and toString() methods. 51 * Tests for proper operation of equals(), hashCode() and toString() methods.
...@@ -56,7 +57,6 @@ public class LeadershipEventTest { ...@@ -56,7 +57,6 @@ public class LeadershipEventTest {
56 .addEqualityGroup(event2, sameAsEvent2) 57 .addEqualityGroup(event2, sameAsEvent2)
57 .addEqualityGroup(event3) 58 .addEqualityGroup(event3)
58 .addEqualityGroup(event4) 59 .addEqualityGroup(event4)
59 - .addEqualityGroup(event5)
60 .testEquals(); 60 .testEquals();
61 } 61 }
62 62
...@@ -65,7 +65,7 @@ public class LeadershipEventTest { ...@@ -65,7 +65,7 @@ public class LeadershipEventTest {
65 */ 65 */
66 @Test 66 @Test
67 public void checkConstruction() { 67 public void checkConstruction() {
68 - assertThat(event1.type(), is(LeadershipEvent.Type.LEADER_ELECTED)); 68 + assertThat(event1.type(), is(LeadershipEvent.Type.LEADER_CHANGED));
69 assertThat(event1.subject(), is(lead1)); 69 assertThat(event1.subject(), is(lead1));
70 70
71 assertThat(event2.time(), is(time)); 71 assertThat(event2.time(), is(time));
......
...@@ -18,7 +18,6 @@ package org.onosproject.cluster; ...@@ -18,7 +18,6 @@ package org.onosproject.cluster;
18 import java.util.List; 18 import java.util.List;
19 import java.util.Map; 19 import java.util.Map;
20 import java.util.Set; 20 import java.util.Set;
21 -import java.util.concurrent.CompletableFuture;
22 21
23 /** 22 /**
24 * Test adapter for leadership service. 23 * Test adapter for leadership service.
...@@ -41,13 +40,12 @@ public class LeadershipServiceAdapter implements LeadershipService { ...@@ -41,13 +40,12 @@ public class LeadershipServiceAdapter implements LeadershipService {
41 } 40 }
42 41
43 @Override 42 @Override
44 - public CompletableFuture<Leadership> runForLeadership(String path) { 43 + public Leadership runForLeadership(String path) {
45 return null; 44 return null;
46 } 45 }
47 46
48 @Override 47 @Override
49 - public CompletableFuture<Void> withdraw(String path) { 48 + public void withdraw(String path) {
50 - return null;
51 } 49 }
52 50
53 @Override 51 @Override
...@@ -74,14 +72,4 @@ public class LeadershipServiceAdapter implements LeadershipService { ...@@ -74,14 +72,4 @@ public class LeadershipServiceAdapter implements LeadershipService {
74 public List<NodeId> getCandidates(String path) { 72 public List<NodeId> getCandidates(String path) {
75 return null; 73 return null;
76 } 74 }
77 -
78 - @Override
79 - public boolean stepdown(String path) {
80 - return false;
81 - }
82 -
83 - @Override
84 - public boolean makeTopCandidate(String path, NodeId nodeId) {
85 - return false;
86 - }
87 } 75 }
...\ No newline at end of file ...\ No newline at end of file
......
...@@ -15,12 +15,13 @@ ...@@ -15,12 +15,13 @@
15 */ 15 */
16 package org.onosproject.cluster; 16 package org.onosproject.cluster;
17 17
18 +import java.util.Arrays;
19 +
18 import org.junit.Test; 20 import org.junit.Test;
19 21
20 import com.google.common.collect.ImmutableList; 22 import com.google.common.collect.ImmutableList;
21 import com.google.common.testing.EqualsTester; 23 import com.google.common.testing.EqualsTester;
22 24
23 -import static org.hamcrest.Matchers.contains;
24 import static org.hamcrest.Matchers.hasSize; 25 import static org.hamcrest.Matchers.hasSize;
25 import static org.hamcrest.Matchers.is; 26 import static org.hamcrest.Matchers.is;
26 import static org.junit.Assert.assertThat; 27 import static org.junit.Assert.assertThat;
...@@ -31,16 +32,14 @@ import static org.junit.Assert.assertThat; ...@@ -31,16 +32,14 @@ import static org.junit.Assert.assertThat;
31 public class LeadershipTest { 32 public class LeadershipTest {
32 private final NodeId node1 = new NodeId("1"); 33 private final NodeId node1 = new NodeId("1");
33 private final NodeId node2 = new NodeId("2"); 34 private final NodeId node2 = new NodeId("2");
34 - private final Leadership lead1 = new Leadership("topic1", node1, 1L, 2L); 35 + private final Leadership lead1 = new Leadership("topic1", new Leader(node1, 1L, 2L), Arrays.asList(node1));
35 - private final Leadership sameAsLead1 = new Leadership("topic1", node1, 1L, 2L); 36 + private final Leadership sameAsLead1 = new Leadership("topic1", new Leader(node1, 1L, 2L), Arrays.asList(node1));
36 - private final Leadership lead2 = new Leadership("topic2", node1, 1L, 2L); 37 + private final Leadership lead2 = new Leadership("topic2", new Leader(node1, 1L, 2L), Arrays.asList(node1));
37 - private final Leadership lead3 = new Leadership("topic1", node1, 2L, 2L); 38 + private final Leadership lead3 = new Leadership("topic1", new Leader(node1, 2L, 2L), Arrays.asList(node1));
38 - private final Leadership lead4 = new Leadership("topic1", node1, 3L, 2L); 39 + private final Leadership lead4 = new Leadership("topic1", new Leader(node1, 3L, 2L), Arrays.asList(node1));
39 - private final Leadership lead5 = new Leadership("topic1", node1, 3L, 3L); 40 + private final Leadership lead5 = new Leadership("topic1", new Leader(node1, 3L, 3L), Arrays.asList(node1));
40 - private final Leadership lead6 = new Leadership("topic1", node1, 41 + private final Leadership lead6 = new Leadership("topic1", new Leader(node2, 1L, 2L), Arrays.asList(node2, node1));
41 - ImmutableList.of(node2), 1L, 2L); 42 + private final Leadership lead7 = new Leadership("topic1", null, ImmutableList.of());
42 - private final Leadership lead7 = new Leadership("topic1",
43 - ImmutableList.of(node2), 1L, 2L);
44 43
45 /** 44 /**
46 * Tests for proper operation of equals(), hashCode() and toString() methods. 45 * Tests for proper operation of equals(), hashCode() and toString() methods.
...@@ -64,12 +63,10 @@ public class LeadershipTest { ...@@ -64,12 +63,10 @@ public class LeadershipTest {
64 */ 63 */
65 @Test 64 @Test
66 public void checkConstruction() { 65 public void checkConstruction() {
67 - assertThat(lead6.electedTime(), is(2L)); 66 + assertThat(lead6.leader(), is(new Leader(node2, 1L, 2L)));
68 - assertThat(lead6.epoch(), is(1L));
69 - assertThat(lead6.leader(), is(node1));
70 assertThat(lead6.topic(), is("topic1")); 67 assertThat(lead6.topic(), is("topic1"));
71 - assertThat(lead6.candidates(), hasSize(1)); 68 + assertThat(lead6.candidates(), hasSize(2));
72 - assertThat(lead6.candidates(), contains(node2)); 69 + assertThat(lead6.candidates().get(1), is(node1));
70 + assertThat(lead6.candidates().get(0), is(node2));
73 } 71 }
74 -
75 } 72 }
......
...@@ -17,20 +17,22 @@ package org.onosproject.store.trivial; ...@@ -17,20 +17,22 @@ package org.onosproject.store.trivial;
17 17
18 import static com.google.common.base.Preconditions.checkArgument; 18 import static com.google.common.base.Preconditions.checkArgument;
19 19
20 +import java.util.Arrays;
20 import java.util.List; 21 import java.util.List;
21 import java.util.Map; 22 import java.util.Map;
22 import java.util.Map.Entry; 23 import java.util.Map.Entry;
23 import java.util.Set; 24 import java.util.Set;
24 -import java.util.concurrent.CompletableFuture;
25 import java.util.concurrent.ConcurrentHashMap; 25 import java.util.concurrent.ConcurrentHashMap;
26 import java.util.concurrent.CopyOnWriteArraySet; 26 import java.util.concurrent.CopyOnWriteArraySet;
27 import java.util.stream.Collectors; 27 import java.util.stream.Collectors;
28 28
29 +import org.apache.felix.scr.annotations.Activate;
29 import org.apache.felix.scr.annotations.Component; 30 import org.apache.felix.scr.annotations.Component;
30 import org.apache.felix.scr.annotations.Reference; 31 import org.apache.felix.scr.annotations.Reference;
31 import org.apache.felix.scr.annotations.ReferenceCardinality; 32 import org.apache.felix.scr.annotations.ReferenceCardinality;
32 import org.apache.felix.scr.annotations.Service; 33 import org.apache.felix.scr.annotations.Service;
33 import org.onosproject.cluster.ClusterService; 34 import org.onosproject.cluster.ClusterService;
35 +import org.onosproject.cluster.Leader;
34 import org.onosproject.cluster.Leadership; 36 import org.onosproject.cluster.Leadership;
35 import org.onosproject.cluster.LeadershipEvent; 37 import org.onosproject.cluster.LeadershipEvent;
36 import org.onosproject.cluster.LeadershipEvent.Type; 38 import org.onosproject.cluster.LeadershipEvent.Type;
...@@ -53,8 +55,15 @@ public class SimpleLeadershipManager implements LeadershipService { ...@@ -53,8 +55,15 @@ public class SimpleLeadershipManager implements LeadershipService {
53 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) 55 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
54 private ClusterService clusterService; 56 private ClusterService clusterService;
55 57
58 + private NodeId localNodeId;
59 +
56 private Map<String, Boolean> elections = new ConcurrentHashMap<>(); 60 private Map<String, Boolean> elections = new ConcurrentHashMap<>();
57 61
62 + @Activate
63 + public void activate() {
64 + localNodeId = clusterService.getLocalNode().id();
65 + }
66 +
58 @Override 67 @Override
59 public NodeId getLeader(String path) { 68 public NodeId getLeader(String path) {
60 return elections.get(path) ? clusterService.getLocalNode().id() : null; 69 return elections.get(path) ? clusterService.getLocalNode().id() : null;
...@@ -63,7 +72,8 @@ public class SimpleLeadershipManager implements LeadershipService { ...@@ -63,7 +72,8 @@ public class SimpleLeadershipManager implements LeadershipService {
63 @Override 72 @Override
64 public Leadership getLeadership(String path) { 73 public Leadership getLeadership(String path) {
65 checkArgument(path != null); 74 checkArgument(path != null);
66 - return elections.get(path) ? new Leadership(path, clusterService.getLocalNode().id(), 0, 0) : null; 75 + return elections.get(path) ?
76 + new Leadership(path, new Leader(localNodeId, 0, 0), Arrays.asList(localNodeId)) : null;
67 } 77 }
68 78
69 @Override 79 @Override
...@@ -77,23 +87,22 @@ public class SimpleLeadershipManager implements LeadershipService { ...@@ -77,23 +87,22 @@ public class SimpleLeadershipManager implements LeadershipService {
77 } 87 }
78 88
79 @Override 89 @Override
80 - public CompletableFuture<Leadership> runForLeadership(String path) { 90 + public Leadership runForLeadership(String path) {
81 elections.put(path, true); 91 elections.put(path, true);
92 + Leadership leadership = new Leadership(path, new Leader(localNodeId, 0, 0), Arrays.asList(localNodeId));
82 for (LeadershipEventListener listener : listeners) { 93 for (LeadershipEventListener listener : listeners) {
83 - listener.event(new LeadershipEvent(Type.LEADER_ELECTED, 94 + listener.event(new LeadershipEvent(Type.LEADER_AND_CANDIDATES_CHANGED, leadership));
84 - new Leadership(path, clusterService.getLocalNode().id(), 0, 0)));
85 } 95 }
86 - return CompletableFuture.completedFuture(new Leadership(path, clusterService.getLocalNode().id(), 0, 0)); 96 + return leadership;
87 } 97 }
88 98
89 @Override 99 @Override
90 - public CompletableFuture<Void> withdraw(String path) { 100 + public void withdraw(String path) {
91 elections.remove(path); 101 elections.remove(path);
92 for (LeadershipEventListener listener : listeners) { 102 for (LeadershipEventListener listener : listeners) {
93 - listener.event(new LeadershipEvent(Type.LEADER_BOOTED, 103 + listener.event(new LeadershipEvent(Type.LEADER_AND_CANDIDATES_CHANGED,
94 - new Leadership(path, clusterService.getLocalNode().id(), 0, 0))); 104 + new Leadership(path, null, Arrays.asList())));
95 } 105 }
96 - return CompletableFuture.completedFuture(null);
97 } 106 }
98 107
99 @Override 108 @Override
...@@ -122,14 +131,4 @@ public class SimpleLeadershipManager implements LeadershipService { ...@@ -122,14 +131,4 @@ public class SimpleLeadershipManager implements LeadershipService {
122 public List<NodeId> getCandidates(String path) { 131 public List<NodeId> getCandidates(String path) {
123 return null; 132 return null;
124 } 133 }
125 -
126 - @Override
127 - public boolean stepdown(String path) {
128 - throw new UnsupportedOperationException();
129 - }
130 -
131 - @Override
132 - public boolean makeTopCandidate(String path, NodeId nodeId) {
133 - throw new UnsupportedOperationException();
134 - }
135 } 134 }
......
1 +/*
2 + * Copyright 2016 Open Networking Laboratory
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +package org.onosproject.cluster.impl;
17 +
18 +import static org.slf4j.LoggerFactory.getLogger;
19 +
20 +import java.util.Map;
21 +import java.util.concurrent.Executors;
22 +import java.util.concurrent.ScheduledExecutorService;
23 +import java.util.concurrent.TimeUnit;
24 +
25 +import org.apache.felix.scr.annotations.Activate;
26 +import org.apache.felix.scr.annotations.Component;
27 +import org.apache.felix.scr.annotations.Deactivate;
28 +import org.apache.felix.scr.annotations.Reference;
29 +import org.apache.felix.scr.annotations.ReferenceCardinality;
30 +import org.apache.felix.scr.annotations.Service;
31 +import org.onlab.util.Tools;
32 +import org.onosproject.cluster.ClusterService;
33 +import org.onosproject.cluster.ControllerNode;
34 +import org.onosproject.cluster.Leadership;
35 +import org.onosproject.cluster.LeadershipAdminService;
36 +import org.onosproject.cluster.LeadershipEvent;
37 +import org.onosproject.cluster.LeadershipEventListener;
38 +import org.onosproject.cluster.LeadershipService;
39 +import org.onosproject.cluster.LeadershipStore;
40 +import org.onosproject.cluster.LeadershipStoreDelegate;
41 +import org.onosproject.cluster.NodeId;
42 +import org.onosproject.event.AbstractListenerManager;
43 +import org.slf4j.Logger;
44 +
45 +import com.google.common.collect.Maps;
46 +
47 +/**
48 + * Implementation of {@link LeadershipService} and {@link LeadershipAdminService}.
49 + */
50 +@Component(immediate = true)
51 +@Service
52 +public class LeadershipManager
53 + extends AbstractListenerManager<LeadershipEvent, LeadershipEventListener>
54 + implements LeadershipService, LeadershipAdminService {
55 +
56 + private final Logger log = getLogger(getClass());
57 +
58 + private LeadershipStoreDelegate delegate = this::post;
59 +
60 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
61 + protected ClusterService clusterService;
62 +
63 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
64 + protected LeadershipStore store;
65 +
66 + private NodeId localNodeId;
67 +
68 + private final ScheduledExecutorService deadlockDetector =
69 + Executors.newSingleThreadScheduledExecutor(Tools.groupedThreads("onos/leadership", ""));
70 +
71 + @Activate
72 + public void activate() {
73 + localNodeId = clusterService.getLocalNode().id();
74 + store.setDelegate(delegate);
75 + eventDispatcher.addSink(LeadershipEvent.class, listenerRegistry);
76 + deadlockDetector.scheduleWithFixedDelay(() -> clusterService.getNodes()
77 + .stream()
78 + .map(ControllerNode::id)
79 + .filter(id -> clusterService.getState(id) != ControllerNode.State.ACTIVE)
80 + .forEach(this::unregister), 0, 2, TimeUnit.SECONDS);
81 + log.info("Started");
82 + }
83 +
84 + @Deactivate
85 + public void deactivate() {
86 + deadlockDetector.shutdown();
87 + Maps.filterValues(store.getLeaderships(), v -> v.candidates().contains(localNodeId))
88 + .keySet()
89 + .forEach(this::withdraw);
90 + store.unsetDelegate(delegate);
91 + eventDispatcher.removeSink(LeadershipEvent.class);
92 + log.info("Stopped");
93 + }
94 +
95 + @Override
96 + public Leadership getLeadership(String topic) {
97 + return store.getLeadership(topic);
98 + }
99 +
100 + @Override
101 + public Leadership runForLeadership(String topic) {
102 + return store.addRegistration(topic);
103 + }
104 +
105 + @Override
106 + public void withdraw(String topic) {
107 + store.removeRegistration(topic);
108 + }
109 +
110 + @Override
111 + public Map<String, Leadership> getLeaderBoard() {
112 + return store.getLeaderships();
113 + }
114 +
115 + @Override
116 + public boolean transferLeadership(String topic, NodeId to) {
117 + return store.moveLeadership(topic, to);
118 + }
119 +
120 + @Override
121 + public void unregister(NodeId nodeId) {
122 + store.removeRegistration(nodeId);
123 + }
124 +
125 + @Override
126 + public boolean promoteToTopOfCandidateList(String topic, NodeId nodeId) {
127 + return store.makeTopCandidate(topic, nodeId);
128 + }
129 +}
1 +/*
2 + * Copyright 2016 Open Networking Laboratory
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +package org.onosproject.store.cluster.impl;
17 +
18 +import static org.slf4j.LoggerFactory.getLogger;
19 +
20 +import java.util.ArrayList;
21 +import java.util.List;
22 +import java.util.Map;
23 +import java.util.stream.Collectors;
24 +
25 +import org.apache.felix.scr.annotations.Activate;
26 +import org.apache.felix.scr.annotations.Component;
27 +import org.apache.felix.scr.annotations.Deactivate;
28 +import org.apache.felix.scr.annotations.Reference;
29 +import org.apache.felix.scr.annotations.ReferenceCardinality;
30 +import org.apache.felix.scr.annotations.Service;
31 +import org.onosproject.cluster.ClusterService;
32 +import org.onosproject.cluster.Leader;
33 +import org.onosproject.cluster.Leadership;
34 +import org.onosproject.cluster.LeadershipEvent;
35 +import org.onosproject.cluster.LeadershipStore;
36 +import org.onosproject.cluster.LeadershipStoreDelegate;
37 +import org.onosproject.cluster.NodeId;
38 +import org.onosproject.store.AbstractStore;
39 +import org.onosproject.store.serializers.KryoNamespaces;
40 +import org.onosproject.store.service.ConsistentMap;
41 +import org.onosproject.store.service.MapEventListener;
42 +import org.onosproject.store.service.Serializer;
43 +import org.onosproject.store.service.StorageService;
44 +import org.onosproject.store.service.Versioned;
45 +import org.slf4j.Logger;
46 +
47 +import com.google.common.base.MoreObjects;
48 +import com.google.common.base.Objects;
49 +import com.google.common.collect.ImmutableList;
50 +import com.google.common.collect.ImmutableMap;
51 +import com.google.common.collect.ImmutableSet;
52 +import com.google.common.collect.Maps;
53 +import com.google.common.collect.Sets;
54 +
55 +/**
56 + * Implementation of {@code LeadershipStore} backed by {@link ConsistentMap}.
57 + */
58 +@Service
59 +@Component(immediate = true, enabled = true)
60 +public class DistributedLeadershipStore
61 + extends AbstractStore<LeadershipEvent, LeadershipStoreDelegate>
62 + implements LeadershipStore {
63 +
64 + private static final Logger log = getLogger(DistributedLeadershipStore.class);
65 +
66 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
67 + protected ClusterService clusterService;
68 +
69 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
70 + protected StorageService storageService;
71 +
72 + protected NodeId localNodeId;
73 + protected ConsistentMap<String, InternalLeadership> leadershipMap;
74 + private final MapEventListener<String, InternalLeadership> leadershipChangeListener =
75 + event -> {
76 + Leadership oldValue = InternalLeadership.toLeadership(Versioned.valueOrNull(event.oldValue()));
77 + Leadership newValue = InternalLeadership.toLeadership(Versioned.valueOrNull(event.newValue()));
78 + boolean leaderChanged =
79 + !Objects.equal(oldValue == null ? null : oldValue.leader(), newValue.leader());
80 + boolean candidatesChanged =
81 + !Sets.symmetricDifference(Sets.newHashSet(oldValue == null ?
82 + ImmutableSet.<NodeId>of() : oldValue.candidates()),
83 + Sets.newHashSet(newValue.candidates())).isEmpty();
84 + LeadershipEvent.Type eventType = null;
85 + if (leaderChanged && candidatesChanged) {
86 + eventType = LeadershipEvent.Type.LEADER_AND_CANDIDATES_CHANGED;
87 + }
88 + if (leaderChanged && !candidatesChanged) {
89 + eventType = LeadershipEvent.Type.LEADER_CHANGED;
90 + }
91 + if (!leaderChanged && candidatesChanged) {
92 + eventType = LeadershipEvent.Type.CANDIDATES_CHANGED;
93 + }
94 + notifyDelegate(new LeadershipEvent(eventType, newValue));
95 + };
96 +
97 + @Activate
98 + public void activate() {
99 + localNodeId = clusterService.getLocalNode().id();
100 + leadershipMap = storageService.<String, InternalLeadership>consistentMapBuilder()
101 + .withName("onos-leadership")
102 + .withPartitionsDisabled()
103 + .withRelaxedReadConsistency()
104 + .withSerializer(Serializer.using(KryoNamespaces.API, InternalLeadership.class))
105 + .build();
106 + leadershipMap.addListener(leadershipChangeListener);
107 + log.info("Started");
108 + }
109 +
110 + @Deactivate
111 + public void deactivate() {
112 + leadershipMap.removeListener(leadershipChangeListener);
113 + log.info("Stopped");
114 + }
115 +
116 + @Override
117 + public Leadership addRegistration(String topic) {
118 + Versioned<InternalLeadership> internalLeadership = leadershipMap.computeIf(topic,
119 + v -> v == null || !v.candidates().contains(localNodeId),
120 + (k, v) -> {
121 + if (v == null || v.candidates().isEmpty()) {
122 + return new InternalLeadership(topic,
123 + localNodeId,
124 + v == null ? 1 : v.term() + 1,
125 + System.currentTimeMillis(),
126 + ImmutableList.of(localNodeId));
127 + }
128 + List<NodeId> newCandidates = new ArrayList<>(v.candidates());
129 + newCandidates.add(localNodeId);
130 + return new InternalLeadership(topic, v.leader(), v.term(), v.termStartTime(), newCandidates);
131 + });
132 + return InternalLeadership.toLeadership(Versioned.valueOrNull(internalLeadership));
133 + }
134 +
135 + @Override
136 + public void removeRegistration(String topic) {
137 + removeRegistration(topic, localNodeId);
138 + }
139 +
140 + private void removeRegistration(String topic, NodeId nodeId) {
141 + leadershipMap.computeIf(topic,
142 + v -> v != null && v.candidates().contains(nodeId),
143 + (k, v) -> {
144 + List<NodeId> newCandidates = v.candidates()
145 + .stream()
146 + .filter(id -> !nodeId.equals(id))
147 + .collect(Collectors.toList());
148 + NodeId newLeader = nodeId.equals(v.leader()) ?
149 + newCandidates.size() > 0 ? newCandidates.get(0) : null : v.leader();
150 + long newTerm = newLeader == null || Objects.equal(newLeader, v.leader()) ?
151 + v.term() : v.term() + 1;
152 + long newTermStartTime = newLeader == null || Objects.equal(newLeader, v.leader()) ?
153 + v.termStartTime() : System.currentTimeMillis();
154 + return new InternalLeadership(topic, newLeader, newTerm, newTermStartTime, newCandidates);
155 + });
156 + }
157 +
158 + @Override
159 + public void removeRegistration(NodeId nodeId) {
160 + leadershipMap.entrySet()
161 + .stream()
162 + .filter(e -> e.getValue().value().candidates().contains(nodeId))
163 + .map(e -> e.getKey())
164 + .forEach(topic -> this.removeRegistration(topic, nodeId));
165 + }
166 +
167 + @Override
168 + public boolean moveLeadership(String topic, NodeId toNodeId) {
169 + Versioned<InternalLeadership> internalLeadership = leadershipMap.computeIf(topic,
170 + v -> v != null &&
171 + v.candidates().contains(toNodeId) &&
172 + !Objects.equal(v.leader(), toNodeId),
173 + (k, v) -> {
174 + List<NodeId> newCandidates = new ArrayList<>();
175 + newCandidates.add(toNodeId);
176 + newCandidates.addAll(v.candidates()
177 + .stream()
178 + .filter(id -> !toNodeId.equals(id))
179 + .collect(Collectors.toList()));
180 + return new InternalLeadership(topic,
181 + toNodeId,
182 + v.term() + 1,
183 + System.currentTimeMillis(),
184 + newCandidates);
185 + });
186 + return Objects.equal(toNodeId, Versioned.valueOrNull(internalLeadership).leader());
187 + }
188 +
189 + @Override
190 + public boolean makeTopCandidate(String topic, NodeId nodeId) {
191 + Versioned<InternalLeadership> internalLeadership = leadershipMap.computeIf(topic,
192 + v -> v != null &&
193 + v.candidates().contains(nodeId) &&
194 + !v.candidates().get(0).equals(nodeId),
195 + (k, v) -> {
196 + List<NodeId> newCandidates = new ArrayList<>();
197 + newCandidates.add(nodeId);
198 + newCandidates.addAll(v.candidates()
199 + .stream()
200 + .filter(id -> !nodeId.equals(id))
201 + .collect(Collectors.toList()));
202 + return new InternalLeadership(topic,
203 + v.leader(),
204 + v.term(),
205 + System.currentTimeMillis(),
206 + newCandidates);
207 + });
208 + return internalLeadership != null && nodeId.equals(internalLeadership.value().candidates().get(0));
209 + }
210 +
211 + @Override
212 + public Leadership getLeadership(String topic) {
213 + return InternalLeadership.toLeadership(Versioned.valueOrNull(leadershipMap.get(topic)));
214 + }
215 +
216 + @Override
217 + public Map<String, Leadership> getLeaderships() {
218 + Map<String, Leadership> leaderships = Maps.newHashMap();
219 + leadershipMap.entrySet().forEach(e -> {
220 + leaderships.put(e.getKey(), e.getValue().value().asLeadership());
221 + });
222 + return ImmutableMap.copyOf(leaderships);
223 + }
224 +
225 + private static class InternalLeadership {
226 + private final String topic;
227 + private final NodeId leader;
228 + private final long term;
229 + private final long termStartTime;
230 + private final List<NodeId> candidates;
231 +
232 + public InternalLeadership(String topic,
233 + NodeId leader,
234 + long term,
235 + long termStartTime,
236 + List<NodeId> candidates) {
237 + this.topic = topic;
238 + this.leader = leader;
239 + this.term = term;
240 + this.termStartTime = termStartTime;
241 + this.candidates = ImmutableList.copyOf(candidates);
242 + }
243 +
244 + public NodeId leader() {
245 + return this.leader;
246 + }
247 +
248 + public long term() {
249 + return term;
250 + }
251 +
252 + public long termStartTime() {
253 + return termStartTime;
254 + }
255 +
256 + public List<NodeId> candidates() {
257 + return candidates;
258 + }
259 +
260 + public Leadership asLeadership() {
261 + return new Leadership(topic, leader == null ?
262 + null : new Leader(leader, term, termStartTime), candidates);
263 + }
264 +
265 + public static Leadership toLeadership(InternalLeadership internalLeadership) {
266 + return internalLeadership == null ? null : internalLeadership.asLeadership();
267 + }
268 +
269 + @Override
270 + public String toString() {
271 + return MoreObjects.toStringHelper(getClass())
272 + .add("leader", leader)
273 + .add("term", term)
274 + .add("termStartTime", termStartTime)
275 + .add("candidates", candidates)
276 + .toString();
277 + }
278 + }
279 +}
...@@ -21,8 +21,6 @@ import org.apache.felix.scr.annotations.Deactivate; ...@@ -21,8 +21,6 @@ import org.apache.felix.scr.annotations.Deactivate;
21 import org.apache.felix.scr.annotations.Reference; 21 import org.apache.felix.scr.annotations.Reference;
22 import org.apache.felix.scr.annotations.ReferenceCardinality; 22 import org.apache.felix.scr.annotations.ReferenceCardinality;
23 import org.apache.felix.scr.annotations.Service; 23 import org.apache.felix.scr.annotations.Service;
24 -import org.onosproject.cluster.ClusterEvent;
25 -import org.onosproject.cluster.ClusterEventListener;
26 import org.onosproject.cluster.ClusterService; 24 import org.onosproject.cluster.ClusterService;
27 import org.onosproject.cluster.ControllerNode; 25 import org.onosproject.cluster.ControllerNode;
28 import org.onosproject.cluster.Leadership; 26 import org.onosproject.cluster.Leadership;
...@@ -76,7 +74,6 @@ public class IntentPartitionManager implements IntentPartitionService { ...@@ -76,7 +74,6 @@ public class IntentPartitionManager implements IntentPartitionService {
76 74
77 private ListenerRegistry<IntentPartitionEvent, IntentPartitionEventListener> listenerRegistry; 75 private ListenerRegistry<IntentPartitionEvent, IntentPartitionEventListener> listenerRegistry;
78 private LeadershipEventListener leaderListener = new InternalLeadershipListener(); 76 private LeadershipEventListener leaderListener = new InternalLeadershipListener();
79 - private ClusterEventListener clusterListener = new InternalClusterEventListener();
80 77
81 private ScheduledExecutorService executor = Executors 78 private ScheduledExecutorService executor = Executors
82 .newScheduledThreadPool(1); 79 .newScheduledThreadPool(1);
...@@ -84,7 +81,6 @@ public class IntentPartitionManager implements IntentPartitionService { ...@@ -84,7 +81,6 @@ public class IntentPartitionManager implements IntentPartitionService {
84 @Activate 81 @Activate
85 public void activate() { 82 public void activate() {
86 leadershipService.addListener(leaderListener); 83 leadershipService.addListener(leaderListener);
87 - clusterService.addListener(clusterListener);
88 84
89 listenerRegistry = new ListenerRegistry<>(); 85 listenerRegistry = new ListenerRegistry<>();
90 eventDispatcher.addSink(IntentPartitionEvent.class, listenerRegistry); 86 eventDispatcher.addSink(IntentPartitionEvent.class, listenerRegistry);
...@@ -103,7 +99,6 @@ public class IntentPartitionManager implements IntentPartitionService { ...@@ -103,7 +99,6 @@ public class IntentPartitionManager implements IntentPartitionService {
103 99
104 eventDispatcher.removeSink(IntentPartitionEvent.class); 100 eventDispatcher.removeSink(IntentPartitionEvent.class);
105 leadershipService.removeListener(leaderListener); 101 leadershipService.removeListener(leaderListener);
106 - clusterService.removeListener(clusterListener);
107 } 102 }
108 103
109 /** 104 /**
...@@ -180,7 +175,7 @@ public class IntentPartitionManager implements IntentPartitionService { ...@@ -180,7 +175,7 @@ public class IntentPartitionManager implements IntentPartitionService {
180 175
181 List<Leadership> myPartitions = leadershipService.getLeaderBoard().values() 176 List<Leadership> myPartitions = leadershipService.getLeaderBoard().values()
182 .stream() 177 .stream()
183 - .filter(l -> clusterService.getLocalNode().id().equals(l.leader())) 178 + .filter(l -> clusterService.getLocalNode().id().equals(l.leaderNodeId()))
184 .filter(l -> l.topic().startsWith(ELECTION_PREFIX)) 179 .filter(l -> l.topic().startsWith(ELECTION_PREFIX))
185 .collect(Collectors.toList()); 180 .collect(Collectors.toList());
186 181
...@@ -220,24 +215,16 @@ public class IntentPartitionManager implements IntentPartitionService { ...@@ -220,24 +215,16 @@ public class IntentPartitionManager implements IntentPartitionService {
220 public void event(LeadershipEvent event) { 215 public void event(LeadershipEvent event) {
221 Leadership leadership = event.subject(); 216 Leadership leadership = event.subject();
222 217
223 - if (Objects.equals(leadership.leader(), clusterService.getLocalNode().id()) && 218 + if (Objects.equals(leadership.leaderNodeId(), clusterService.getLocalNode().id()) &&
224 leadership.topic().startsWith(ELECTION_PREFIX)) { 219 leadership.topic().startsWith(ELECTION_PREFIX)) {
225 220
226 - // See if we need to let some partitions go
227 - scheduleRebalance(0);
228 -
229 eventDispatcher.post(new IntentPartitionEvent(IntentPartitionEvent.Type.LEADER_CHANGED, 221 eventDispatcher.post(new IntentPartitionEvent(IntentPartitionEvent.Type.LEADER_CHANGED,
230 leadership.topic())); 222 leadership.topic()));
231 } 223 }
232 - }
233 - }
234 224
235 - private final class InternalClusterEventListener implements 225 + if (event.type() == LeadershipEvent.Type.CANDIDATES_CHANGED) {
236 - ClusterEventListener {
237 -
238 - @Override
239 - public void event(ClusterEvent event) {
240 scheduleRebalance(0); 226 scheduleRebalance(0);
241 } 227 }
242 } 228 }
229 + }
243 } 230 }
......
...@@ -16,7 +16,6 @@ ...@@ -16,7 +16,6 @@
16 package org.onosproject.store.mastership.impl; 16 package org.onosproject.store.mastership.impl;
17 17
18 import static org.onlab.util.Tools.groupedThreads; 18 import static org.onlab.util.Tools.groupedThreads;
19 -import static org.onlab.util.Tools.futureGetOrElse;
20 import static org.onosproject.mastership.MastershipEvent.Type.BACKUPS_CHANGED; 19 import static org.onosproject.mastership.MastershipEvent.Type.BACKUPS_CHANGED;
21 import static org.onosproject.mastership.MastershipEvent.Type.MASTER_CHANGED; 20 import static org.onosproject.mastership.MastershipEvent.Type.MASTER_CHANGED;
22 import static org.slf4j.LoggerFactory.getLogger; 21 import static org.slf4j.LoggerFactory.getLogger;
...@@ -43,6 +42,7 @@ import org.apache.felix.scr.annotations.Service; ...@@ -43,6 +42,7 @@ import org.apache.felix.scr.annotations.Service;
43 import org.onlab.util.KryoNamespace; 42 import org.onlab.util.KryoNamespace;
44 import org.onosproject.cluster.ClusterService; 43 import org.onosproject.cluster.ClusterService;
45 import org.onosproject.cluster.Leadership; 44 import org.onosproject.cluster.Leadership;
45 +import org.onosproject.cluster.LeadershipAdminService;
46 import org.onosproject.cluster.LeadershipEvent; 46 import org.onosproject.cluster.LeadershipEvent;
47 import org.onosproject.cluster.LeadershipEventListener; 47 import org.onosproject.cluster.LeadershipEventListener;
48 import org.onosproject.cluster.LeadershipService; 48 import org.onosproject.cluster.LeadershipService;
...@@ -63,9 +63,9 @@ import org.onosproject.store.serializers.StoreSerializer; ...@@ -63,9 +63,9 @@ import org.onosproject.store.serializers.StoreSerializer;
63 import org.slf4j.Logger; 63 import org.slf4j.Logger;
64 64
65 import com.google.common.base.Objects; 65 import com.google.common.base.Objects;
66 +import com.google.common.collect.ImmutableList;
66 import com.google.common.collect.Lists; 67 import com.google.common.collect.Lists;
67 import com.google.common.collect.Maps; 68 import com.google.common.collect.Maps;
68 -import com.google.common.collect.Sets;
69 69
70 /** 70 /**
71 * Implementation of the MastershipStore on top of Leadership Service. 71 * Implementation of the MastershipStore on top of Leadership Service.
...@@ -82,18 +82,18 @@ public class ConsistentDeviceMastershipStore ...@@ -82,18 +82,18 @@ public class ConsistentDeviceMastershipStore
82 protected LeadershipService leadershipService; 82 protected LeadershipService leadershipService;
83 83
84 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) 84 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
85 + protected LeadershipAdminService leadershipAdminService;
86 +
87 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
85 protected ClusterService clusterService; 88 protected ClusterService clusterService;
86 89
87 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) 90 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
88 protected ClusterCommunicationService clusterCommunicator; 91 protected ClusterCommunicationService clusterCommunicator;
89 92
90 private NodeId localNodeId; 93 private NodeId localNodeId;
91 - private final Set<DeviceId> connectedDevices = Sets.newHashSet();
92 94
93 private static final MessageSubject ROLE_RELINQUISH_SUBJECT = 95 private static final MessageSubject ROLE_RELINQUISH_SUBJECT =
94 new MessageSubject("mastership-store-device-role-relinquish"); 96 new MessageSubject("mastership-store-device-role-relinquish");
95 - private static final MessageSubject TRANSITION_FROM_MASTER_TO_STANDBY_SUBJECT =
96 - new MessageSubject("mastership-store-device-mastership-relinquish");
97 97
98 private static final Pattern DEVICE_MASTERSHIP_TOPIC_PATTERN = 98 private static final Pattern DEVICE_MASTERSHIP_TOPIC_PATTERN =
99 Pattern.compile("device:(.*)"); 99 Pattern.compile("device:(.*)");
...@@ -132,11 +132,6 @@ public class ConsistentDeviceMastershipStore ...@@ -132,11 +132,6 @@ public class ConsistentDeviceMastershipStore
132 this::relinquishLocalRole, 132 this::relinquishLocalRole,
133 SERIALIZER::encode, 133 SERIALIZER::encode,
134 messageHandlingExecutor); 134 messageHandlingExecutor);
135 - clusterCommunicator.addSubscriber(TRANSITION_FROM_MASTER_TO_STANDBY_SUBJECT,
136 - SERIALIZER::decode,
137 - this::transitionFromMasterToStandby,
138 - SERIALIZER::encode,
139 - messageHandlingExecutor);
140 localNodeId = clusterService.getLocalNode().id(); 135 localNodeId = clusterService.getLocalNode().id();
141 leadershipService.addListener(leadershipEventListener); 136 leadershipService.addListener(leadershipEventListener);
142 137
...@@ -146,7 +141,6 @@ public class ConsistentDeviceMastershipStore ...@@ -146,7 +141,6 @@ public class ConsistentDeviceMastershipStore
146 @Deactivate 141 @Deactivate
147 public void deactivate() { 142 public void deactivate() {
148 clusterCommunicator.removeSubscriber(ROLE_RELINQUISH_SUBJECT); 143 clusterCommunicator.removeSubscriber(ROLE_RELINQUISH_SUBJECT);
149 - clusterCommunicator.removeSubscriber(TRANSITION_FROM_MASTER_TO_STANDBY_SUBJECT);
150 messageHandlingExecutor.shutdown(); 144 messageHandlingExecutor.shutdown();
151 transferExecutor.shutdown(); 145 transferExecutor.shutdown();
152 leadershipService.removeListener(leadershipEventListener); 146 leadershipService.removeListener(leadershipEventListener);
...@@ -159,12 +153,9 @@ public class ConsistentDeviceMastershipStore ...@@ -159,12 +153,9 @@ public class ConsistentDeviceMastershipStore
159 checkArgument(deviceId != null, DEVICE_ID_NULL); 153 checkArgument(deviceId != null, DEVICE_ID_NULL);
160 154
161 String leadershipTopic = createDeviceMastershipTopic(deviceId); 155 String leadershipTopic = createDeviceMastershipTopic(deviceId);
162 - connectedDevices.add(deviceId); 156 + Leadership leadership = leadershipService.runForLeadership(leadershipTopic);
163 - return leadershipService.runForLeadership(leadershipTopic) 157 + return CompletableFuture.completedFuture(localNodeId.equals(leadership.leaderNodeId())
164 - .thenApply(leadership -> { 158 + ? MastershipRole.MASTER : MastershipRole.STANDBY);
165 - return Objects.equal(localNodeId, leadership.leader())
166 - ? MastershipRole.MASTER : MastershipRole.STANDBY;
167 - });
168 } 159 }
169 160
170 @Override 161 @Override
...@@ -173,20 +164,19 @@ public class ConsistentDeviceMastershipStore ...@@ -173,20 +164,19 @@ public class ConsistentDeviceMastershipStore
173 checkArgument(deviceId != null, DEVICE_ID_NULL); 164 checkArgument(deviceId != null, DEVICE_ID_NULL);
174 165
175 String leadershipTopic = createDeviceMastershipTopic(deviceId); 166 String leadershipTopic = createDeviceMastershipTopic(deviceId);
176 - NodeId leader = leadershipService.getLeader(leadershipTopic); 167 + Leadership leadership = leadershipService.getLeadership(leadershipTopic);
177 - if (Objects.equal(nodeId, leader)) { 168 + NodeId leader = leadership == null ? null : leadership.leaderNodeId();
178 - return MastershipRole.MASTER; 169 + List<NodeId> candidates = leadership == null ?
179 - } 170 + ImmutableList.of() : ImmutableList.copyOf(leadership.candidates());
180 - return leadershipService.getCandidates(leadershipTopic).contains(nodeId) ? 171 + return Objects.equal(nodeId, leader) ?
181 - MastershipRole.STANDBY : MastershipRole.NONE; 172 + MastershipRole.MASTER : candidates.contains(nodeId) ? MastershipRole.STANDBY : MastershipRole.NONE;
182 } 173 }
183 174
184 @Override 175 @Override
185 public NodeId getMaster(DeviceId deviceId) { 176 public NodeId getMaster(DeviceId deviceId) {
186 checkArgument(deviceId != null, DEVICE_ID_NULL); 177 checkArgument(deviceId != null, DEVICE_ID_NULL);
187 178
188 - String leadershipTopic = createDeviceMastershipTopic(deviceId); 179 + return leadershipService.getLeader(createDeviceMastershipTopic(deviceId));
189 - return leadershipService.getLeader(leadershipTopic);
190 } 180 }
191 181
192 @Override 182 @Override
...@@ -194,8 +184,7 @@ public class ConsistentDeviceMastershipStore ...@@ -194,8 +184,7 @@ public class ConsistentDeviceMastershipStore
194 checkArgument(deviceId != null, DEVICE_ID_NULL); 184 checkArgument(deviceId != null, DEVICE_ID_NULL);
195 185
196 Map<NodeId, MastershipRole> roles = Maps.newHashMap(); 186 Map<NodeId, MastershipRole> roles = Maps.newHashMap();
197 - clusterService 187 + clusterService.getNodes()
198 - .getNodes()
199 .forEach((node) -> roles.put(node.id(), getRole(node.id(), deviceId))); 188 .forEach((node) -> roles.put(node.id(), getRole(node.id(), deviceId)));
200 189
201 NodeId master = null; 190 NodeId master = null;
...@@ -233,30 +222,10 @@ public class ConsistentDeviceMastershipStore ...@@ -233,30 +222,10 @@ public class ConsistentDeviceMastershipStore
233 checkArgument(nodeId != null, NODE_ID_NULL); 222 checkArgument(nodeId != null, NODE_ID_NULL);
234 checkArgument(deviceId != null, DEVICE_ID_NULL); 223 checkArgument(deviceId != null, DEVICE_ID_NULL);
235 224
236 - NodeId currentMaster = getMaster(deviceId);
237 - if (nodeId.equals(currentMaster)) {
238 - return CompletableFuture.completedFuture(null);
239 - } else {
240 String leadershipTopic = createDeviceMastershipTopic(deviceId); 225 String leadershipTopic = createDeviceMastershipTopic(deviceId);
241 - List<NodeId> candidates = leadershipService.getCandidates(leadershipTopic); 226 + if (leadershipAdminService.promoteToTopOfCandidateList(leadershipTopic, nodeId)) {
242 - if (candidates.isEmpty()) { 227 + transferExecutor.schedule(() -> leadershipAdminService.transferLeadership(leadershipTopic, nodeId),
243 - return CompletableFuture.completedFuture(null); 228 + WAIT_BEFORE_MASTERSHIP_HANDOFF_MILLIS, TimeUnit.MILLISECONDS);
244 - }
245 - if (leadershipService.makeTopCandidate(leadershipTopic, nodeId)) {
246 - CompletableFuture<MastershipEvent> result = new CompletableFuture<>();
247 - // There is brief wait before we step down from mastership.
248 - // This is to ensure any work that happens when standby preference
249 - // order changes can complete. For example: flow entries need to be backed
250 - // to the new top standby (ONOS-1883)
251 - // FIXME: This potentially introduces a race-condition.
252 - // Right now role changes are only forced via CLI.
253 - transferExecutor.schedule(() -> {
254 - result.complete(transitionFromMasterToStandby(deviceId));
255 - }, WAIT_BEFORE_MASTERSHIP_HANDOFF_MILLIS, TimeUnit.MILLISECONDS);
256 - return result;
257 - } else {
258 - log.warn("Failed to promote {} to mastership for {}", nodeId, deviceId);
259 - }
260 } 229 }
261 return CompletableFuture.completedFuture(null); 230 return CompletableFuture.completedFuture(null);
262 } 231 }
...@@ -267,7 +236,7 @@ public class ConsistentDeviceMastershipStore ...@@ -267,7 +236,7 @@ public class ConsistentDeviceMastershipStore
267 236
268 String leadershipTopic = createDeviceMastershipTopic(deviceId); 237 String leadershipTopic = createDeviceMastershipTopic(deviceId);
269 Leadership leadership = leadershipService.getLeadership(leadershipTopic); 238 Leadership leadership = leadershipService.getLeadership(leadershipTopic);
270 - return leadership != null ? MastershipTerm.of(leadership.leader(), leadership.epoch()) : null; 239 + return leadership != null ? MastershipTerm.of(leadership.leaderNodeId(), leadership.leader().term()) : null;
271 } 240 }
272 241
273 @Override 242 @Override
...@@ -318,71 +287,44 @@ public class ConsistentDeviceMastershipStore ...@@ -318,71 +287,44 @@ public class ConsistentDeviceMastershipStore
318 private CompletableFuture<MastershipEvent> relinquishLocalRole(DeviceId deviceId) { 287 private CompletableFuture<MastershipEvent> relinquishLocalRole(DeviceId deviceId) {
319 checkArgument(deviceId != null, DEVICE_ID_NULL); 288 checkArgument(deviceId != null, DEVICE_ID_NULL);
320 289
321 - // Check if this node is can be managed by this node.
322 - if (!connectedDevices.contains(deviceId)) {
323 - return CompletableFuture.completedFuture(null);
324 - }
325 -
326 String leadershipTopic = createDeviceMastershipTopic(deviceId); 290 String leadershipTopic = createDeviceMastershipTopic(deviceId);
327 - NodeId currentLeader = leadershipService.getLeader(leadershipTopic); 291 + if (!leadershipService.getCandidates(leadershipTopic).contains(localNodeId)) {
328 - 292 + return CompletableFuture.completedFuture(null);
329 - MastershipEvent.Type eventType = Objects.equal(currentLeader, localNodeId)
330 - ? MastershipEvent.Type.MASTER_CHANGED
331 - : MastershipEvent.Type.BACKUPS_CHANGED;
332 -
333 - connectedDevices.remove(deviceId);
334 - return leadershipService.withdraw(leadershipTopic)
335 - .thenApply(v -> new MastershipEvent(eventType, deviceId, getNodes(deviceId)));
336 } 293 }
337 - 294 + MastershipEvent.Type eventType = localNodeId.equals(leadershipService.getLeader(leadershipTopic)) ?
338 - private MastershipEvent transitionFromMasterToStandby(DeviceId deviceId) { 295 + MastershipEvent.Type.MASTER_CHANGED : MastershipEvent.Type.BACKUPS_CHANGED;
339 - checkArgument(deviceId != null, DEVICE_ID_NULL); 296 + leadershipService.withdraw(leadershipTopic);
340 - 297 + return CompletableFuture.completedFuture(new MastershipEvent(eventType, deviceId, getNodes(deviceId)));
341 - NodeId currentMaster = getMaster(deviceId);
342 - if (currentMaster == null) {
343 - return null;
344 } 298 }
345 299
346 - if (!currentMaster.equals(localNodeId)) { 300 + @Override
347 - log.info("Forwarding request to relinquish " 301 + public void relinquishAllRole(NodeId nodeId) {
348 - + "mastership for device {} to {}", deviceId, currentMaster); 302 + // Noop. LeadershipService already takes care of detecting and purging stale locks.
349 - return futureGetOrElse(clusterCommunicator.sendAndReceive(
350 - deviceId,
351 - TRANSITION_FROM_MASTER_TO_STANDBY_SUBJECT,
352 - SERIALIZER::encode,
353 - SERIALIZER::decode,
354 - currentMaster), null);
355 } 303 }
356 304
357 - return leadershipService.stepdown(createDeviceMastershipTopic(deviceId)) 305 + private class InternalDeviceMastershipEventListener implements LeadershipEventListener {
358 - ? new MastershipEvent(MastershipEvent.Type.MASTER_CHANGED, deviceId, getNodes(deviceId)) : null;
359 - }
360 306
361 @Override 307 @Override
362 - public void relinquishAllRole(NodeId nodeId) { 308 + public boolean isRelevant(LeadershipEvent event) {
363 - // Noop. LeadershipService already takes care of detecting and purging deadlocks. 309 + Leadership leadership = event.subject();
310 + return isDeviceMastershipTopic(leadership.topic());
364 } 311 }
365 312
366 - private class InternalDeviceMastershipEventListener implements LeadershipEventListener {
367 @Override 313 @Override
368 public void event(LeadershipEvent event) { 314 public void event(LeadershipEvent event) {
369 Leadership leadership = event.subject(); 315 Leadership leadership = event.subject();
370 - if (!isDeviceMastershipTopic(leadership.topic())) {
371 - return;
372 - }
373 DeviceId deviceId = extractDeviceIdFromTopic(leadership.topic()); 316 DeviceId deviceId = extractDeviceIdFromTopic(leadership.topic());
317 + RoleInfo roleInfo = getNodes(deviceId);
374 switch (event.type()) { 318 switch (event.type()) {
375 - case LEADER_ELECTED: 319 + case LEADER_AND_CANDIDATES_CHANGED:
376 - notifyDelegate(new MastershipEvent(MASTER_CHANGED, deviceId, getNodes(deviceId))); 320 + notifyDelegate(new MastershipEvent(BACKUPS_CHANGED, deviceId, roleInfo));
321 + notifyDelegate(new MastershipEvent(MASTER_CHANGED, deviceId, roleInfo));
377 break; 322 break;
378 - case LEADER_REELECTED: 323 + case LEADER_CHANGED:
379 - // There is no concept of leader re-election in the new distributed leadership manager. 324 + notifyDelegate(new MastershipEvent(MASTER_CHANGED, deviceId, roleInfo));
380 - throw new IllegalStateException("Unexpected event type");
381 - case LEADER_BOOTED:
382 - notifyDelegate(new MastershipEvent(MASTER_CHANGED, deviceId, getNodes(deviceId)));
383 break; 325 break;
384 case CANDIDATES_CHANGED: 326 case CANDIDATES_CHANGED:
385 - notifyDelegate(new MastershipEvent(BACKUPS_CHANGED, deviceId, getNodes(deviceId))); 327 + notifyDelegate(new MastershipEvent(BACKUPS_CHANGED, deviceId, roleInfo));
386 break; 328 break;
387 default: 329 default:
388 return; 330 return;
...@@ -407,5 +349,4 @@ public class ConsistentDeviceMastershipStore ...@@ -407,5 +349,4 @@ public class ConsistentDeviceMastershipStore
407 Matcher m = DEVICE_MASTERSHIP_TOPIC_PATTERN.matcher(topic); 349 Matcher m = DEVICE_MASTERSHIP_TOPIC_PATTERN.matcher(topic);
408 return m.matches(); 350 return m.matches();
409 } 351 }
410 -
411 } 352 }
...\ No newline at end of file ...\ No newline at end of file
......
...@@ -22,6 +22,7 @@ import org.onlab.packet.IpAddress; ...@@ -22,6 +22,7 @@ import org.onlab.packet.IpAddress;
22 import org.onosproject.cluster.ClusterServiceAdapter; 22 import org.onosproject.cluster.ClusterServiceAdapter;
23 import org.onosproject.cluster.ControllerNode; 23 import org.onosproject.cluster.ControllerNode;
24 import org.onosproject.cluster.DefaultControllerNode; 24 import org.onosproject.cluster.DefaultControllerNode;
25 +import org.onosproject.cluster.Leader;
25 import org.onosproject.cluster.Leadership; 26 import org.onosproject.cluster.Leadership;
26 import org.onosproject.cluster.LeadershipEvent; 27 import org.onosproject.cluster.LeadershipEvent;
27 import org.onosproject.cluster.LeadershipEventListener; 28 import org.onosproject.cluster.LeadershipEventListener;
...@@ -31,13 +32,12 @@ import org.onosproject.cluster.NodeId; ...@@ -31,13 +32,12 @@ import org.onosproject.cluster.NodeId;
31 import org.onosproject.common.event.impl.TestEventDispatcher; 32 import org.onosproject.common.event.impl.TestEventDispatcher;
32 import org.onosproject.net.intent.Key; 33 import org.onosproject.net.intent.Key;
33 34
35 +import java.util.Arrays;
34 import java.util.HashMap; 36 import java.util.HashMap;
35 import java.util.HashSet; 37 import java.util.HashSet;
36 import java.util.Map; 38 import java.util.Map;
37 import java.util.Objects; 39 import java.util.Objects;
38 import java.util.Set; 40 import java.util.Set;
39 -import java.util.concurrent.CompletableFuture;
40 -
41 import static junit.framework.TestCase.assertFalse; 41 import static junit.framework.TestCase.assertFalse;
42 import static org.easymock.EasyMock.anyObject; 42 import static org.easymock.EasyMock.anyObject;
43 import static org.easymock.EasyMock.anyString; 43 import static org.easymock.EasyMock.anyString;
...@@ -55,9 +55,10 @@ import static org.junit.Assert.assertTrue; ...@@ -55,9 +55,10 @@ import static org.junit.Assert.assertTrue;
55 public class IntentPartitionManagerTest { 55 public class IntentPartitionManagerTest {
56 56
57 private final LeadershipEvent event 57 private final LeadershipEvent event
58 - = new LeadershipEvent(LeadershipEvent.Type.LEADER_ELECTED, 58 + = new LeadershipEvent(LeadershipEvent.Type.CANDIDATES_CHANGED,
59 new Leadership(ELECTION_PREFIX + "0", 59 new Leadership(ELECTION_PREFIX + "0",
60 - MY_NODE_ID, 0, 0)); 60 + new Leader(MY_NODE_ID, 0, 0),
61 + Arrays.asList(MY_NODE_ID, OTHER_NODE_ID)));
61 62
62 private static final NodeId MY_NODE_ID = new NodeId("local"); 63 private static final NodeId MY_NODE_ID = new NodeId("local");
63 private static final NodeId OTHER_NODE_ID = new NodeId("other"); 64 private static final NodeId OTHER_NODE_ID = new NodeId("other");
...@@ -78,7 +79,7 @@ public class IntentPartitionManagerTest { ...@@ -78,7 +79,7 @@ public class IntentPartitionManagerTest {
78 expectLastCall().andDelegateTo(new TestLeadershipService()); 79 expectLastCall().andDelegateTo(new TestLeadershipService());
79 for (int i = 0; i < IntentPartitionManager.NUM_PARTITIONS; i++) { 80 for (int i = 0; i < IntentPartitionManager.NUM_PARTITIONS; i++) {
80 expect(leadershipService.runForLeadership(ELECTION_PREFIX + i)) 81 expect(leadershipService.runForLeadership(ELECTION_PREFIX + i))
81 - .andReturn(CompletableFuture.completedFuture(null)) 82 + .andReturn(null)
82 .times(1); 83 .times(1);
83 } 84 }
84 85
...@@ -105,7 +106,9 @@ public class IntentPartitionManagerTest { ...@@ -105,7 +106,9 @@ public class IntentPartitionManagerTest {
105 expect(leadershipService.getLeader(ELECTION_PREFIX + i)) 106 expect(leadershipService.getLeader(ELECTION_PREFIX + i))
106 .andReturn(MY_NODE_ID).anyTimes(); 107 .andReturn(MY_NODE_ID).anyTimes();
107 leaderBoard.put(ELECTION_PREFIX + i, 108 leaderBoard.put(ELECTION_PREFIX + i,
108 - new Leadership(ELECTION_PREFIX + i, MY_NODE_ID, 0, 0)); 109 + new Leadership(ELECTION_PREFIX + i,
110 + new Leader(MY_NODE_ID, 0, 0),
111 + Arrays.asList(MY_NODE_ID)));
109 } 112 }
110 113
111 for (int i = numMine; i < IntentPartitionManager.NUM_PARTITIONS; i++) { 114 for (int i = numMine; i < IntentPartitionManager.NUM_PARTITIONS; i++) {
...@@ -113,7 +116,9 @@ public class IntentPartitionManagerTest { ...@@ -113,7 +116,9 @@ public class IntentPartitionManagerTest {
113 .andReturn(OTHER_NODE_ID).anyTimes(); 116 .andReturn(OTHER_NODE_ID).anyTimes();
114 117
115 leaderBoard.put(ELECTION_PREFIX + i, 118 leaderBoard.put(ELECTION_PREFIX + i,
116 - new Leadership(ELECTION_PREFIX + i, OTHER_NODE_ID, 0, 0)); 119 + new Leadership(ELECTION_PREFIX + i,
120 + new Leader(OTHER_NODE_ID, 0, 0),
121 + Arrays.asList(OTHER_NODE_ID)));
117 } 122 }
118 123
119 expect(leadershipService.getLeaderBoard()).andReturn(leaderBoard).anyTimes(); 124 expect(leadershipService.getLeaderBoard()).andReturn(leaderBoard).anyTimes();
...@@ -131,7 +136,7 @@ public class IntentPartitionManagerTest { ...@@ -131,7 +136,7 @@ public class IntentPartitionManagerTest {
131 136
132 for (int i = 0; i < IntentPartitionManager.NUM_PARTITIONS; i++) { 137 for (int i = 0; i < IntentPartitionManager.NUM_PARTITIONS; i++) {
133 expect(leadershipService.runForLeadership(ELECTION_PREFIX + i)) 138 expect(leadershipService.runForLeadership(ELECTION_PREFIX + i))
134 - .andReturn(CompletableFuture.completedFuture(null)) 139 + .andReturn(null)
135 .times(1); 140 .times(1);
136 } 141 }
137 142
...@@ -200,9 +205,8 @@ public class IntentPartitionManagerTest { ...@@ -200,9 +205,8 @@ public class IntentPartitionManagerTest {
200 // We have all the partitions so we'll need to relinquish some 205 // We have all the partitions so we'll need to relinquish some
201 setUpLeadershipService(IntentPartitionManager.NUM_PARTITIONS); 206 setUpLeadershipService(IntentPartitionManager.NUM_PARTITIONS);
202 207
203 - expect(leadershipService.withdraw(anyString())) 208 + leadershipService.withdraw(anyString());
204 - .andReturn(CompletableFuture.completedFuture(null)) 209 + expectLastCall().times(7);
205 - .times(7);
206 210
207 replay(leadershipService); 211 replay(leadershipService);
208 212
......
1 -/*
2 - * Copyright 2015 Open Networking Laboratory
3 - *
4 - * Licensed under the Apache License, Version 2.0 (the "License");
5 - * you may not use this file except in compliance with the License.
6 - * You may obtain a copy of the License at
7 - *
8 - * http://www.apache.org/licenses/LICENSE-2.0
9 - *
10 - * Unless required by applicable law or agreed to in writing, software
11 - * distributed under the License is distributed on an "AS IS" BASIS,
12 - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 - * See the License for the specific language governing permissions and
14 - * limitations under the License.
15 - */
16 -package org.onosproject.store.primitives.impl;
17 -
18 -import com.google.common.collect.ImmutableList;
19 -import com.google.common.collect.ImmutableMap;
20 -import com.google.common.collect.Iterables;
21 -import com.google.common.collect.Lists;
22 -import com.google.common.collect.MapDifference;
23 -import com.google.common.collect.Maps;
24 -import com.google.common.collect.Sets;
25 -
26 -import org.apache.commons.lang.math.RandomUtils;
27 -import org.apache.felix.scr.annotations.Activate;
28 -import org.apache.felix.scr.annotations.Component;
29 -import org.apache.felix.scr.annotations.Deactivate;
30 -import org.apache.felix.scr.annotations.Reference;
31 -import org.apache.felix.scr.annotations.ReferenceCardinality;
32 -import org.apache.felix.scr.annotations.Service;
33 -import org.onosproject.cluster.ClusterEvent;
34 -import org.onosproject.cluster.ClusterEvent.Type;
35 -import org.onosproject.cluster.ClusterEventListener;
36 -import org.onosproject.cluster.ClusterService;
37 -import org.onosproject.cluster.Leadership;
38 -import org.onosproject.cluster.LeadershipEvent;
39 -import org.onosproject.cluster.LeadershipEventListener;
40 -import org.onosproject.cluster.LeadershipService;
41 -import org.onosproject.cluster.NodeId;
42 -import org.onosproject.event.ListenerRegistry;
43 -import org.onosproject.event.EventDeliveryService;
44 -import org.onosproject.store.cluster.messaging.ClusterCommunicationService;
45 -import org.onosproject.store.serializers.KryoNamespaces;
46 -import org.onosproject.store.service.ConsistentMap;
47 -import org.onosproject.store.service.ConsistentMapException;
48 -import org.onosproject.store.service.MapEvent;
49 -import org.onosproject.store.service.Serializer;
50 -import org.onosproject.store.service.StorageService;
51 -import org.onosproject.store.service.Versioned;
52 -import org.slf4j.Logger;
53 -
54 -import java.util.ArrayList;
55 -import java.util.Collections;
56 -import java.util.Map;
57 -import java.util.Map.Entry;
58 -import java.util.Objects;
59 -import java.util.Set;
60 -import java.util.List;
61 -import java.util.concurrent.CancellationException;
62 -import java.util.concurrent.CompletableFuture;
63 -import java.util.concurrent.Executors;
64 -import java.util.concurrent.ScheduledExecutorService;
65 -import java.util.concurrent.TimeUnit;
66 -import java.util.concurrent.atomic.AtomicBoolean;
67 -import java.util.stream.Collectors;
68 -
69 -import static com.google.common.base.Preconditions.checkArgument;
70 -import static org.onlab.util.Tools.groupedThreads;
71 -import static org.slf4j.LoggerFactory.getLogger;
72 -import static org.onosproject.cluster.ControllerNode.State.ACTIVE;
73 -import static org.onosproject.cluster.ControllerNode.State.INACTIVE;
74 -
75 -/**
76 - * Distributed Lock Manager implemented on top of ConsistentMap.
77 - * <p>
78 - * This implementation makes use of ClusterService's failure
79 - * detection capabilities to detect and purge stale locks.
80 - * TODO: Ensure lock safety and liveness.
81 - */
82 -@Component(immediate = true, enabled = true)
83 -@Service
84 -public class DistributedLeadershipManager implements LeadershipService {
85 -
86 - @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
87 - protected StorageService storageService;
88 -
89 - @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
90 - protected ClusterService clusterService;
91 -
92 - @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
93 - protected ClusterCommunicationService clusterCommunicator;
94 -
95 - @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
96 - protected EventDeliveryService eventDispatcher;
97 -
98 - private final Logger log = getLogger(getClass());
99 -
100 - private ScheduledExecutorService electionRunner;
101 - private ScheduledExecutorService lockExecutor;
102 - private ScheduledExecutorService staleLeadershipPurgeExecutor;
103 - private ScheduledExecutorService leadershipRefresher;
104 -
105 - // leader for each topic
106 - private ConsistentMap<String, NodeId> leaderMap;
107 - // list of candidates (includes chosen leader) for each topic
108 - private ConsistentMap<String, List<NodeId>> candidateMap;
109 -
110 - private ListenerRegistry<LeadershipEvent, LeadershipEventListener> listenerRegistry;
111 -
112 - // cached copy of leaderMap
113 - // Note: Map value, Leadership, does not contain proper candidates info
114 - private final Map<String, Leadership> leaderBoard = Maps.newConcurrentMap();
115 - // cached copy of candidateMap
116 - // Note: Map value, Leadership, does not contain proper leader info
117 - private final Map<String, Leadership> candidateBoard = Maps.newConcurrentMap();
118 -
119 - private final ClusterEventListener clusterEventListener = new InternalClusterEventListener();
120 -
121 - private NodeId localNodeId;
122 - private Set<String> activeTopics = Sets.newConcurrentHashSet();
123 - private Map<String, CompletableFuture<Leadership>> pendingFutures = Maps.newConcurrentMap();
124 -
125 - // The actual delay is randomly chosen from the interval [0, WAIT_BEFORE_RETRY_MILLIS)
126 - private static final int WAIT_BEFORE_RETRY_MILLIS = 150;
127 - private static final int DELAY_BETWEEN_LEADER_LOCK_ATTEMPTS_SEC = 2;
128 - private static final int LEADERSHIP_REFRESH_INTERVAL_SEC = 2;
129 - private static final int DELAY_BETWEEN_STALE_LEADERSHIP_PURGE_ATTEMPTS_SEC = 2;
130 -
131 - private final AtomicBoolean staleLeadershipPurgeScheduled = new AtomicBoolean(false);
132 -
133 - private static final Serializer SERIALIZER = Serializer.using(KryoNamespaces.API);
134 -
135 - @Activate
136 - public void activate() {
137 - leaderMap = storageService.<String, NodeId>consistentMapBuilder()
138 - .withName("onos-topic-leaders")
139 - .withSerializer(SERIALIZER)
140 - .withPartitionsDisabled().build();
141 - candidateMap = storageService.<String, List<NodeId>>consistentMapBuilder()
142 - .withName("onos-topic-candidates")
143 - .withSerializer(SERIALIZER)
144 - .withPartitionsDisabled().build();
145 -
146 - leaderMap.addListener(event -> {
147 - log.debug("Received {}", event);
148 - LeadershipEvent.Type leadershipEventType = null;
149 - if (event.type() == MapEvent.Type.INSERT || event.type() == MapEvent.Type.UPDATE) {
150 - leadershipEventType = LeadershipEvent.Type.LEADER_ELECTED;
151 - } else if (event.type() == MapEvent.Type.REMOVE) {
152 - leadershipEventType = LeadershipEvent.Type.LEADER_BOOTED;
153 - }
154 - onLeadershipEvent(new LeadershipEvent(
155 - leadershipEventType,
156 - new Leadership(event.key(),
157 - event.value().value(),
158 - event.value().version(),
159 - event.value().creationTime())));
160 - });
161 -
162 - candidateMap.addListener(event -> {
163 - log.debug("Received {}", event);
164 - if (event.type() != MapEvent.Type.INSERT && event.type() != MapEvent.Type.UPDATE) {
165 - log.error("Entries must not be removed from candidate map");
166 - return;
167 - }
168 - onLeadershipEvent(new LeadershipEvent(
169 - LeadershipEvent.Type.CANDIDATES_CHANGED,
170 - new Leadership(event.key(),
171 - event.value().value(),
172 - event.value().version(),
173 - event.value().creationTime())));
174 - });
175 -
176 - localNodeId = clusterService.getLocalNode().id();
177 -
178 - electionRunner = Executors.newSingleThreadScheduledExecutor(
179 - groupedThreads("onos/store/leadership", "election-runner"));
180 - lockExecutor = Executors.newScheduledThreadPool(
181 - 4, groupedThreads("onos/store/leadership", "election-thread-%d"));
182 - staleLeadershipPurgeExecutor = Executors.newSingleThreadScheduledExecutor(
183 - groupedThreads("onos/store/leadership", "stale-leadership-evictor"));
184 - leadershipRefresher = Executors.newSingleThreadScheduledExecutor(
185 - groupedThreads("onos/store/leadership", "refresh-thread"));
186 -
187 - clusterService.addListener(clusterEventListener);
188 -
189 - electionRunner.scheduleWithFixedDelay(
190 - this::electLeaders, 0, DELAY_BETWEEN_LEADER_LOCK_ATTEMPTS_SEC, TimeUnit.SECONDS);
191 -
192 - leadershipRefresher.scheduleWithFixedDelay(
193 - this::refreshLeaderBoard, 0, LEADERSHIP_REFRESH_INTERVAL_SEC, TimeUnit.SECONDS);
194 -
195 - listenerRegistry = new ListenerRegistry<>();
196 - eventDispatcher.addSink(LeadershipEvent.class, listenerRegistry);
197 -
198 - log.info("Started");
199 - }
200 -
201 - @Deactivate
202 - public void deactivate() {
203 - if (clusterService.getNodes().size() > 1) {
204 - // FIXME: Determine why this takes ~50 seconds to shutdown on a single node!
205 - leaderBoard.forEach((topic, leadership) -> {
206 - if (localNodeId.equals(leadership.leader())) {
207 - withdraw(topic);
208 - }
209 - });
210 - }
211 -
212 - clusterService.removeListener(clusterEventListener);
213 - eventDispatcher.removeSink(LeadershipEvent.class);
214 -
215 - electionRunner.shutdown();
216 - lockExecutor.shutdown();
217 - staleLeadershipPurgeExecutor.shutdown();
218 - leadershipRefresher.shutdown();
219 -
220 - log.info("Stopped");
221 - }
222 -
223 - @Override
224 - public Map<String, Leadership> getLeaderBoard() {
225 - return ImmutableMap.copyOf(leaderBoard);
226 - }
227 -
228 - @Override
229 - public Map<String, List<NodeId>> getCandidates() {
230 - return Maps.toMap(candidateBoard.keySet(), this::getCandidates);
231 - }
232 -
233 - @Override
234 - public List<NodeId> getCandidates(String path) {
235 - Leadership current = candidateBoard.get(path);
236 - return current == null ? ImmutableList.of() : ImmutableList.copyOf(current.candidates());
237 - }
238 -
239 - @Override
240 - public NodeId getLeader(String path) {
241 - Leadership leadership = leaderBoard.get(path);
242 - return leadership != null ? leadership.leader() : null;
243 - }
244 -
245 - @Override
246 - public Leadership getLeadership(String path) {
247 - checkArgument(path != null);
248 - return leaderBoard.get(path);
249 - }
250 -
251 - @Override
252 - public Set<String> ownedTopics(NodeId nodeId) {
253 - checkArgument(nodeId != null);
254 - return leaderBoard.entrySet()
255 - .stream()
256 - .filter(entry -> nodeId.equals(entry.getValue().leader()))
257 - .map(Entry::getKey)
258 - .collect(Collectors.toSet());
259 - }
260 -
261 - @Override
262 - public CompletableFuture<Leadership> runForLeadership(String path) {
263 - log.debug("Running for leadership for topic: {}", path);
264 - CompletableFuture<Leadership> resultFuture = new CompletableFuture<>();
265 - doRunForLeadership(path, resultFuture);
266 - return resultFuture;
267 - }
268 -
269 - private void doRunForLeadership(String path, CompletableFuture<Leadership> future) {
270 - try {
271 - Versioned<List<NodeId>> candidates = candidateMap.computeIf(path,
272 - currentList -> currentList == null || !currentList.contains(localNodeId),
273 - (topic, currentList) -> {
274 - if (currentList == null) {
275 - return ImmutableList.of(localNodeId);
276 - } else {
277 - List<NodeId> newList = Lists.newLinkedList();
278 - newList.addAll(currentList);
279 - newList.add(localNodeId);
280 - return newList;
281 - }
282 - });
283 - log.debug("In the leadership race for topic {} with candidates {}", path, candidates);
284 - activeTopics.add(path);
285 - Leadership leadership = electLeader(path, candidates.value());
286 - if (leadership == null) {
287 - pendingFutures.put(path, future);
288 - } else {
289 - future.complete(leadership);
290 - }
291 - } catch (ConsistentMapException e) {
292 - log.debug("Failed to enter topic leader race for {}. Retrying.", path, e);
293 - rerunForLeadership(path, future);
294 - }
295 - }
296 -
297 - @Override
298 - public CompletableFuture<Void> withdraw(String path) {
299 - activeTopics.remove(path);
300 - CompletableFuture<Void> resultFuture = new CompletableFuture<>();
301 - doWithdraw(path, resultFuture);
302 - return resultFuture;
303 - }
304 -
305 -
306 - private void doWithdraw(String path, CompletableFuture<Void> future) {
307 - if (activeTopics.contains(path)) {
308 - future.completeExceptionally(new CancellationException(String.format("%s is now a active topic", path)));
309 - }
310 - try {
311 - leaderMap.computeIf(path,
312 - localNodeId::equals,
313 - (topic, leader) -> null);
314 - candidateMap.computeIf(path,
315 - candidates -> candidates != null && candidates.contains(localNodeId),
316 - (topic, candidates) -> candidates.stream()
317 - .filter(nodeId -> !localNodeId.equals(nodeId))
318 - .collect(Collectors.toList()));
319 - future.complete(null);
320 - } catch (Exception e) {
321 - log.debug("Failed to verify (and clear) any lock this node might be holding for {}", path, e);
322 - retryWithdraw(path, future);
323 - }
324 - }
325 -
326 - @Override
327 - public boolean stepdown(String path) {
328 - if (!activeTopics.contains(path) || !Objects.equals(localNodeId, getLeader(path))) {
329 - return false;
330 - }
331 -
332 - try {
333 - return leaderMap.computeIf(path,
334 - localNodeId::equals,
335 - (topic, leader) -> null) == null;
336 - } catch (Exception e) {
337 - log.warn("Error executing stepdown for {}", path, e);
338 - }
339 - return false;
340 - }
341 -
342 - @Override
343 - public void addListener(LeadershipEventListener listener) {
344 - listenerRegistry.addListener(listener);
345 - }
346 -
347 - @Override
348 - public void removeListener(LeadershipEventListener listener) {
349 - listenerRegistry.removeListener(listener);
350 - }
351 -
352 - @Override
353 - public boolean makeTopCandidate(String path, NodeId nodeId) {
354 - Versioned<List<NodeId>> candidateList = candidateMap.computeIf(path,
355 - candidates -> candidates != null &&
356 - candidates.contains(nodeId) &&
357 - !nodeId.equals(Iterables.getFirst(candidates, null)),
358 - (topic, candidates) -> {
359 - List<NodeId> updatedCandidates = new ArrayList<>(candidates.size());
360 - updatedCandidates.add(nodeId);
361 - candidates.stream().filter(id -> !nodeId.equals(id)).forEach(updatedCandidates::add);
362 - return updatedCandidates;
363 - });
364 - List<NodeId> candidates = candidateList != null ? candidateList.value() : Collections.emptyList();
365 - return candidates.size() > 0 && nodeId.equals(candidates.get(0));
366 - }
367 -
368 - private Leadership electLeader(String path, List<NodeId> candidates) {
369 - Leadership currentLeadership = getLeadership(path);
370 - if (currentLeadership != null) {
371 - return currentLeadership;
372 - } else {
373 - NodeId topCandidate = candidates
374 - .stream()
375 - .filter(n -> clusterService.getState(n) == ACTIVE)
376 - .findFirst()
377 - .orElse(null);
378 - try {
379 - Versioned<NodeId> leader = localNodeId.equals(topCandidate)
380 - ? leaderMap.computeIfAbsent(path, p -> localNodeId) : leaderMap.get(path);
381 - if (leader != null) {
382 - Leadership newLeadership = new Leadership(path,
383 - leader.value(),
384 - leader.version(),
385 - leader.creationTime());
386 - // Since reads only go through the local copy of leader board, we ought to update it
387 - // first before returning from this method.
388 - // This is to ensure a subsequent read will not read a stale value.
389 - onLeadershipEvent(new LeadershipEvent(LeadershipEvent.Type.LEADER_ELECTED, newLeadership));
390 - return newLeadership;
391 - }
392 - } catch (Exception e) {
393 - log.debug("Failed to elect leader for {}", path, e);
394 - }
395 - }
396 - return null;
397 - }
398 -
399 - private void electLeaders() {
400 - try {
401 - candidateMap.entrySet().forEach(entry -> {
402 - String path = entry.getKey();
403 - Versioned<List<NodeId>> candidates = entry.getValue();
404 - // for active topics, check if this node can become a leader (if it isn't already)
405 - if (activeTopics.contains(path)) {
406 - lockExecutor.submit(() -> {
407 - Leadership leadership = electLeader(path, candidates.value());
408 - if (leadership != null) {
409 - CompletableFuture<Leadership> future = pendingFutures.remove(path);
410 - if (future != null) {
411 - future.complete(leadership);
412 - }
413 - }
414 - });
415 - }
416 - // Raise a CANDIDATES_CHANGED event to force refresh local candidate board
417 - // and also to update local listeners.
418 - // Don't worry about duplicate events as they will be suppressed.
419 - onLeadershipEvent(new LeadershipEvent(LeadershipEvent.Type.CANDIDATES_CHANGED,
420 - new Leadership(path,
421 - candidates.value(),
422 - candidates.version(),
423 - candidates.creationTime())));
424 - });
425 - } catch (Exception e) {
426 - log.debug("Failure electing leaders", e);
427 - }
428 - }
429 -
430 - private void onLeadershipEvent(LeadershipEvent leadershipEvent) {
431 - log.trace("Leadership Event: time = {} type = {} event = {}",
432 - leadershipEvent.time(), leadershipEvent.type(),
433 - leadershipEvent);
434 -
435 - Leadership leadershipUpdate = leadershipEvent.subject();
436 - LeadershipEvent.Type eventType = leadershipEvent.type();
437 - String topic = leadershipUpdate.topic();
438 -
439 - AtomicBoolean updateAccepted = new AtomicBoolean(false);
440 - if (eventType.equals(LeadershipEvent.Type.LEADER_ELECTED)) {
441 - leaderBoard.compute(topic, (k, currentLeadership) -> {
442 - if (currentLeadership == null || currentLeadership.epoch() < leadershipUpdate.epoch()) {
443 - updateAccepted.set(true);
444 - return leadershipUpdate;
445 - }
446 - return currentLeadership;
447 - });
448 - } else if (eventType.equals(LeadershipEvent.Type.LEADER_BOOTED)) {
449 - leaderBoard.compute(topic, (k, currentLeadership) -> {
450 - if (currentLeadership == null || currentLeadership.epoch() <= leadershipUpdate.epoch()) {
451 - updateAccepted.set(true);
452 - // FIXME: Removing entries from leaderboard is not safe and should be visited.
453 - return null;
454 - }
455 - return currentLeadership;
456 - });
457 - } else if (eventType.equals(LeadershipEvent.Type.CANDIDATES_CHANGED)) {
458 - candidateBoard.compute(topic, (k, currentInfo) -> {
459 - if (currentInfo == null || currentInfo.epoch() < leadershipUpdate.epoch()) {
460 - updateAccepted.set(true);
461 - return leadershipUpdate;
462 - }
463 - return currentInfo;
464 - });
465 - } else {
466 - throw new IllegalStateException("Unknown event type.");
467 - }
468 -
469 - if (updateAccepted.get()) {
470 - eventDispatcher.post(leadershipEvent);
471 - }
472 - }
473 -
474 - private void rerunForLeadership(String path, CompletableFuture<Leadership> future) {
475 - lockExecutor.schedule(
476 - () -> doRunForLeadership(path, future),
477 - RandomUtils.nextInt(WAIT_BEFORE_RETRY_MILLIS),
478 - TimeUnit.MILLISECONDS);
479 - }
480 -
481 - private void retryWithdraw(String path, CompletableFuture<Void> future) {
482 - lockExecutor.schedule(
483 - () -> doWithdraw(path, future),
484 - RandomUtils.nextInt(WAIT_BEFORE_RETRY_MILLIS),
485 - TimeUnit.MILLISECONDS);
486 - }
487 -
488 - private void scheduleStaleLeadershipPurge(int afterDelaySec) {
489 - if (staleLeadershipPurgeScheduled.compareAndSet(false, true)) {
490 - staleLeadershipPurgeExecutor.schedule(
491 - this::purgeStaleLeadership,
492 - afterDelaySec,
493 - TimeUnit.SECONDS);
494 - }
495 - }
496 -
497 - /**
498 - * Purges locks held by inactive nodes and evicts inactive nodes from candidacy.
499 - */
500 - private void purgeStaleLeadership() {
501 - AtomicBoolean rerunPurge = new AtomicBoolean(false);
502 - try {
503 - staleLeadershipPurgeScheduled.set(false);
504 - leaderMap.entrySet()
505 - .stream()
506 - .filter(e -> clusterService.getState(e.getValue().value()) == INACTIVE)
507 - .forEach(entry -> {
508 - String path = entry.getKey();
509 - NodeId nodeId = entry.getValue().value();
510 - try {
511 - leaderMap.computeIf(path, nodeId::equals, (topic, leader) -> null);
512 - } catch (Exception e) {
513 - log.debug("Failed to purge stale lock held by {} for {}", nodeId, path, e);
514 - rerunPurge.set(true);
515 - }
516 - });
517 -
518 - candidateMap.entrySet()
519 - .forEach(entry -> {
520 - String path = entry.getKey();
521 - Versioned<List<NodeId>> candidates = entry.getValue();
522 - List<NodeId> candidatesList = candidates != null
523 - ? candidates.value() : Collections.emptyList();
524 - List<NodeId> activeCandidatesList =
525 - candidatesList.stream()
526 - .filter(n -> clusterService.getState(n) == ACTIVE)
527 - .filter(n -> !localNodeId.equals(n) || activeTopics.contains(path))
528 - .collect(Collectors.toList());
529 - if (activeCandidatesList.size() < candidatesList.size()) {
530 - Set<NodeId> removedCandidates =
531 - Sets.difference(Sets.newHashSet(candidatesList),
532 - Sets.newHashSet(activeCandidatesList));
533 - try {
534 - candidateMap.computeIf(path,
535 - c -> c.stream()
536 - .filter(n -> clusterService.getState(n) == INACTIVE)
537 - .count() > 0,
538 - (topic, c) -> c.stream()
539 - .filter(n -> clusterService.getState(n) == ACTIVE)
540 - .filter(n -> !localNodeId.equals(n) ||
541 - activeTopics.contains(path))
542 - .collect(Collectors.toList()));
543 - } catch (Exception e) {
544 - log.debug("Failed to evict inactive candidates {} from "
545 - + "candidate list for {}", removedCandidates, path, e);
546 - rerunPurge.set(true);
547 - }
548 - }
549 - });
550 - } catch (Exception e) {
551 - log.debug("Failure purging state leadership.", e);
552 - rerunPurge.set(true);
553 - }
554 -
555 - if (rerunPurge.get()) {
556 - log.debug("Rescheduling stale leadership purge due to errors encountered in previous run");
557 - scheduleStaleLeadershipPurge(DELAY_BETWEEN_STALE_LEADERSHIP_PURGE_ATTEMPTS_SEC);
558 - }
559 - }
560 -
561 - private void refreshLeaderBoard() {
562 - try {
563 - Map<String, Leadership> newLeaderBoard = Maps.newHashMap();
564 - leaderMap.entrySet().forEach(entry -> {
565 - String path = entry.getKey();
566 - Versioned<NodeId> leader = entry.getValue();
567 - Leadership leadership = new Leadership(path,
568 - leader.value(),
569 - leader.version(),
570 - leader.creationTime());
571 - newLeaderBoard.put(path, leadership);
572 - });
573 -
574 - // first take snapshot of current leader board.
575 - Map<String, Leadership> currentLeaderBoard = ImmutableMap.copyOf(leaderBoard);
576 -
577 - MapDifference<String, Leadership> diff = Maps.difference(currentLeaderBoard, newLeaderBoard);
578 -
579 - // evict stale leaders
580 - diff.entriesOnlyOnLeft().forEach((path, leadership) -> {
581 - log.debug("Evicting {} from leaderboard. It is no longer active leader.", leadership);
582 - onLeadershipEvent(new LeadershipEvent(LeadershipEvent.Type.LEADER_BOOTED, leadership));
583 - });
584 -
585 - // add missing leaders
586 - diff.entriesOnlyOnRight().forEach((path, leadership) -> {
587 - log.debug("Adding {} to leaderboard. It is now the active leader.", leadership);
588 - onLeadershipEvent(new LeadershipEvent(LeadershipEvent.Type.LEADER_ELECTED, leadership));
589 - });
590 -
591 - // add updated leaders
592 - diff.entriesDiffering().forEach((path, difference) -> {
593 - Leadership current = difference.leftValue();
594 - Leadership updated = difference.rightValue();
595 - if (current.epoch() < updated.epoch()) {
596 - log.debug("Updated {} in leaderboard.", updated);
597 - onLeadershipEvent(new LeadershipEvent(LeadershipEvent.Type.LEADER_ELECTED, updated));
598 - }
599 - });
600 - } catch (Exception e) {
601 - log.debug("Failed to refresh leader board", e);
602 - }
603 - }
604 -
605 - private class InternalClusterEventListener implements ClusterEventListener {
606 -
607 - @Override
608 - public void event(ClusterEvent event) {
609 - if (event.type() == Type.INSTANCE_DEACTIVATED || event.type() == Type.INSTANCE_REMOVED) {
610 - scheduleStaleLeadershipPurge(0);
611 - }
612 - }
613 - }
614 -}