홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Yuta HIGUCHI
0c6e1840 1 parent c1957324

Trigger MastershipEvent on no more master case 

Change-Id: Iaac7b7d021802e7470df061dad719dcdf0e4b73e
Inline Side-by-side
Showing 5 changed files with 238 additions and 129 deletions
...@@ -23,6 +23,10 @@ import org.apache.felix.scr.annotations.ReferenceCardinality; ...@@ -23,6 +23,10 @@ import org.apache.felix.scr.annotations.ReferenceCardinality;
23 import org.onlab.onos.cluster.ClusterEvent; 23 import org.onlab.onos.cluster.ClusterEvent;
24 import org.onlab.onos.cluster.ClusterEventListener; 24 import org.onlab.onos.cluster.ClusterEventListener;
25 import org.onlab.onos.cluster.ClusterService; 25 import org.onlab.onos.cluster.ClusterService;
26 +import org.onlab.onos.cluster.NodeId;
27 +import org.onlab.onos.mastership.MastershipEvent;
28 +import org.onlab.onos.mastership.MastershipListener;
29 +import org.onlab.onos.mastership.MastershipService;
26 import org.onlab.onos.net.device.DeviceEvent; 30 import org.onlab.onos.net.device.DeviceEvent;
27 import org.onlab.onos.net.device.DeviceListener; 31 import org.onlab.onos.net.device.DeviceListener;
28 import org.onlab.onos.net.device.DeviceService; 32 import org.onlab.onos.net.device.DeviceService;
...@@ -50,15 +54,20 @@ public class FooComponent { ...@@ -50,15 +54,20 @@ public class FooComponent {
50 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) 54 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
51 protected IntentService intentService; 55 protected IntentService intentService;
52 56
57 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
58 + protected MastershipService mastershipService;
59 +
53 private final ClusterEventListener clusterListener = new InnerClusterListener(); 60 private final ClusterEventListener clusterListener = new InnerClusterListener();
54 private final DeviceListener deviceListener = new InnerDeviceListener(); 61 private final DeviceListener deviceListener = new InnerDeviceListener();
55 private final IntentListener intentListener = new InnerIntentListener(); 62 private final IntentListener intentListener = new InnerIntentListener();
63 + private final MastershipListener mastershipListener = new InnerMastershipListener();
56 64
57 @Activate 65 @Activate
58 public void activate() { 66 public void activate() {
59 clusterService.addListener(clusterListener); 67 clusterService.addListener(clusterListener);
60 deviceService.addListener(deviceListener); 68 deviceService.addListener(deviceListener);
61 intentService.addListener(intentListener); 69 intentService.addListener(intentListener);
70 + mastershipService.addListener(mastershipListener);
62 log.info("Started"); 71 log.info("Started");
63 } 72 }
64 73
...@@ -67,6 +76,7 @@ public class FooComponent { ...@@ -67,6 +76,7 @@ public class FooComponent {
67 clusterService.removeListener(clusterListener); 76 clusterService.removeListener(clusterListener);
68 deviceService.removeListener(deviceListener); 77 deviceService.removeListener(deviceListener);
69 intentService.removeListener(intentListener); 78 intentService.removeListener(intentListener);
79 + mastershipService.removeListener(mastershipListener);
70 log.info("Stopped"); 80 log.info("Stopped");
71 } 81 }
72 82
...@@ -100,6 +110,18 @@ public class FooComponent { ...@@ -100,6 +110,18 @@ public class FooComponent {
100 log.info(message, event.subject()); 110 log.info(message, event.subject());
101 } 111 }
102 } 112 }
113 +
114 + private class InnerMastershipListener implements MastershipListener {
115 + @Override
116 + public void event(MastershipEvent event) {
117 + final NodeId myId = clusterService.getLocalNode().id();
118 + if (myId.equals(event.roleInfo().master())) {
119 + log.info("I have control/I wish you luck {}", event);
120 + } else {
121 + log.info("you have control {}", event);
122 + }
123 + }
124 + }
103 } 125 }
104 126
105 127
......
...@@ -28,6 +28,7 @@ import org.onlab.onos.cluster.DefaultControllerNode; ...@@ -28,6 +28,7 @@ import org.onlab.onos.cluster.DefaultControllerNode;
28 import org.onlab.onos.cluster.NodeId; 28 import org.onlab.onos.cluster.NodeId;
29 import org.onlab.onos.event.impl.TestEventDispatcher; 29 import org.onlab.onos.event.impl.TestEventDispatcher;
30 import org.onlab.onos.mastership.MastershipService; 30 import org.onlab.onos.mastership.MastershipService;
31 +import org.onlab.onos.mastership.MastershipStore;
31 import org.onlab.onos.mastership.MastershipTermService; 32 import org.onlab.onos.mastership.MastershipTermService;
32 import org.onlab.onos.net.DeviceId; 33 import org.onlab.onos.net.DeviceId;
33 import org.onlab.onos.store.trivial.impl.SimpleMastershipStore; 34 import org.onlab.onos.store.trivial.impl.SimpleMastershipStore;
...@@ -57,9 +58,9 @@ public class MastershipManagerTest { ...@@ -57,9 +58,9 @@ public class MastershipManagerTest {
57 public void setUp() { 58 public void setUp() {
58 mgr = new MastershipManager(); 59 mgr = new MastershipManager();
59 service = mgr; 60 service = mgr;
60 - mgr.store = new SimpleMastershipStore();
61 mgr.eventDispatcher = new TestEventDispatcher(); 61 mgr.eventDispatcher = new TestEventDispatcher();
62 mgr.clusterService = new TestClusterService(); 62 mgr.clusterService = new TestClusterService();
63 + mgr.store = new TestSimpleMastershipStore(mgr.clusterService);
63 mgr.activate(); 64 mgr.activate();
64 } 65 }
65 66
...@@ -74,7 +75,8 @@ public class MastershipManagerTest { ...@@ -74,7 +75,8 @@ public class MastershipManagerTest {
74 @Test 75 @Test
75 public void setRole() { 76 public void setRole() {
76 mgr.setRole(NID_OTHER, DEV_MASTER, MASTER); 77 mgr.setRole(NID_OTHER, DEV_MASTER, MASTER);
77 - assertEquals("wrong local role:", STANDBY, mgr.getLocalRole(DEV_MASTER)); 78 + assertEquals("wrong local role:", NONE, mgr.getLocalRole(DEV_MASTER));
79 + assertEquals("wrong obtained role:", STANDBY, mgr.requestRoleFor(DEV_MASTER));
78 80
79 //set to master 81 //set to master
80 mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER); 82 mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
...@@ -182,4 +184,12 @@ public class MastershipManagerTest { ...@@ -182,4 +184,12 @@ public class MastershipManagerTest {
182 } 184 }
183 185
184 } 186 }
187 +
188 + private final class TestSimpleMastershipStore extends SimpleMastershipStore
189 + implements MastershipStore {
190 +
191 + public TestSimpleMastershipStore(ClusterService clusterService) {
192 + super.clusterService = clusterService;
193 + }
194 + }
185 } 195 }
......
...@@ -283,16 +283,15 @@ implements MastershipStore { ...@@ -283,16 +283,15 @@ implements MastershipStore {
283 case MASTER: 283 case MASTER:
284 NodeId newMaster = reelect(nodeId, deviceId, rv); 284 NodeId newMaster = reelect(nodeId, deviceId, rv);
285 rv.reassign(nodeId, NONE, STANDBY); 285 rv.reassign(nodeId, NONE, STANDBY);
286 + updateTerm(deviceId);
286 if (newMaster != null) { 287 if (newMaster != null) {
287 - updateTerm(deviceId);
288 roleMap.put(deviceId, rv); 288 roleMap.put(deviceId, rv);
289 return new MastershipEvent(MASTER_CHANGED, deviceId, rv.roleInfo()); 289 return new MastershipEvent(MASTER_CHANGED, deviceId, rv.roleInfo());
290 } else { 290 } else {
291 // no master candidate 291 // no master candidate
292 roleMap.put(deviceId, rv); 292 roleMap.put(deviceId, rv);
293 - // FIXME: Should there be new event type? 293 + // TODO: Should there be new event type for no MASTER?
294 - // or should we issue null Master event? 294 + return new MastershipEvent(MASTER_CHANGED, deviceId, rv.roleInfo());
295 - return null;
296 } 295 }
297 case STANDBY: 296 case STANDBY:
298 return null; 297 return null;
......
...@@ -29,8 +29,13 @@ import java.util.concurrent.atomic.AtomicInteger; ...@@ -29,8 +29,13 @@ import java.util.concurrent.atomic.AtomicInteger;
29 import org.apache.felix.scr.annotations.Activate; 29 import org.apache.felix.scr.annotations.Activate;
30 import org.apache.felix.scr.annotations.Component; 30 import org.apache.felix.scr.annotations.Component;
31 import org.apache.felix.scr.annotations.Deactivate; 31 import org.apache.felix.scr.annotations.Deactivate;
32 +import org.apache.felix.scr.annotations.Reference;
33 +import org.apache.felix.scr.annotations.ReferenceCardinality;
32 import org.apache.felix.scr.annotations.Service; 34 import org.apache.felix.scr.annotations.Service;
35 +import org.onlab.onos.cluster.ClusterEventListener;
36 +import org.onlab.onos.cluster.ClusterService;
33 import org.onlab.onos.cluster.ControllerNode; 37 import org.onlab.onos.cluster.ControllerNode;
38 +import org.onlab.onos.cluster.ControllerNode.State;
34 import org.onlab.onos.cluster.DefaultControllerNode; 39 import org.onlab.onos.cluster.DefaultControllerNode;
35 import org.onlab.onos.cluster.NodeId; 40 import org.onlab.onos.cluster.NodeId;
36 import org.onlab.onos.cluster.RoleInfo; 41 import org.onlab.onos.cluster.RoleInfo;
...@@ -44,7 +49,8 @@ import org.onlab.onos.store.AbstractStore; ...@@ -44,7 +49,8 @@ import org.onlab.onos.store.AbstractStore;
44 import org.onlab.packet.IpAddress; 49 import org.onlab.packet.IpAddress;
45 import org.slf4j.Logger; 50 import org.slf4j.Logger;
46 51
47 -import com.google.common.collect.Lists; 52 +import com.google.common.collect.ImmutableList;
53 +import com.google.common.collect.ImmutableSet;
48 54
49 import static org.onlab.onos.mastership.MastershipEvent.Type.*; 55 import static org.onlab.onos.mastership.MastershipEvent.Type.*;
50 56
...@@ -60,23 +66,65 @@ public class SimpleMastershipStore ...@@ -60,23 +66,65 @@ public class SimpleMastershipStore
60 66
61 private final Logger log = getLogger(getClass()); 67 private final Logger log = getLogger(getClass());
62 68
63 - public static final IpAddress LOCALHOST = IpAddress.valueOf("127.0.0.1");
64 -
65 private static final int NOTHING = 0; 69 private static final int NOTHING = 0;
66 private static final int INIT = 1; 70 private static final int INIT = 1;
67 71
68 - private ControllerNode instance = 72 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
69 - new DefaultControllerNode(new NodeId("local"), LOCALHOST); 73 + protected ClusterService clusterService;
70 74
71 //devices mapped to their masters, to emulate multiple nodes 75 //devices mapped to their masters, to emulate multiple nodes
72 protected final Map<DeviceId, NodeId> masterMap = new HashMap<>(); 76 protected final Map<DeviceId, NodeId> masterMap = new HashMap<>();
73 //emulate backups with pile of nodes 77 //emulate backups with pile of nodes
74 - protected final Set<NodeId> backups = new HashSet<>(); 78 + protected final Map<DeviceId, List<NodeId>> backups = new HashMap<>();
75 //terms 79 //terms
76 protected final Map<DeviceId, AtomicInteger> termMap = new HashMap<>(); 80 protected final Map<DeviceId, AtomicInteger> termMap = new HashMap<>();
77 81
78 @Activate 82 @Activate
79 public void activate() { 83 public void activate() {
84 + if (clusterService == null) {
85 + // just for ease of unit test
86 + final ControllerNode instance =
87 + new DefaultControllerNode(new NodeId("local"),
88 + IpAddress.valueOf("127.0.0.1"));
89 +
90 + clusterService = new ClusterService() {
91 +
92 + @Override
93 + public ControllerNode getLocalNode() {
94 + return instance;
95 + }
96 +
97 + @Override
98 + public Set<ControllerNode> getNodes() {
99 + return ImmutableSet.of(instance);
100 + }
101 +
102 + @Override
103 + public ControllerNode getNode(NodeId nodeId) {
104 + if (instance.id().equals(nodeId)) {
105 + return instance;
106 + }
107 + return null;
108 + }
109 +
110 + @Override
111 + public State getState(NodeId nodeId) {
112 + if (instance.id().equals(nodeId)) {
113 + return State.ACTIVE;
114 + } else {
115 + return State.INACTIVE;
116 + }
117 + }
118 +
119 + @Override
120 + public void addListener(ClusterEventListener listener) {
121 + }
122 +
123 + @Override
124 + public void removeListener(ClusterEventListener listener) {
125 + }
126 + };
127 + }
80 log.info("Started"); 128 log.info("Started");
81 } 129 }
82 130
...@@ -86,31 +134,27 @@ public class SimpleMastershipStore ...@@ -86,31 +134,27 @@ public class SimpleMastershipStore
86 } 134 }
87 135
88 @Override 136 @Override
89 - public MastershipEvent setMaster(NodeId nodeId, DeviceId deviceId) { 137 + public synchronized MastershipEvent setMaster(NodeId nodeId, DeviceId deviceId) {
90 - MastershipRole role = getRole(nodeId, deviceId);
91 138
92 - synchronized (this) { 139 + MastershipRole role = getRole(nodeId, deviceId);
93 - switch (role) { 140 + switch (role) {
94 - case MASTER: 141 + case MASTER:
95 - return null; 142 + // no-op
96 - case STANDBY: 143 + return null;
97 - masterMap.put(deviceId, nodeId); 144 + case STANDBY:
98 - termMap.get(deviceId).incrementAndGet(); 145 + case NONE:
99 - backups.add(nodeId); 146 + NodeId prevMaster = masterMap.put(deviceId, nodeId);
100 - break; 147 + incrementTerm(deviceId);
101 - case NONE: 148 + removeFromBackups(deviceId, nodeId);
102 - masterMap.put(deviceId, nodeId); 149 + addToBackup(deviceId, prevMaster);
103 - termMap.put(deviceId, new AtomicInteger(INIT)); 150 + break;
104 - backups.add(nodeId); 151 + default:
105 - break; 152 + log.warn("unknown Mastership Role {}", role);
106 - default: 153 + return null;
107 - log.warn("unknown Mastership Role {}", role);
108 - return null;
109 - }
110 } 154 }
111 155
112 return new MastershipEvent(MASTER_CHANGED, deviceId, 156 return new MastershipEvent(MASTER_CHANGED, deviceId,
113 - new RoleInfo(nodeId, Lists.newLinkedList(backups))); 157 + getNodes(deviceId));
114 } 158 }
115 159
116 @Override 160 @Override
...@@ -118,12 +162,11 @@ public class SimpleMastershipStore ...@@ -118,12 +162,11 @@ public class SimpleMastershipStore
118 return masterMap.get(deviceId); 162 return masterMap.get(deviceId);
119 } 163 }
120 164
165 + // synchronized for atomic read
121 @Override 166 @Override
122 - public RoleInfo getNodes(DeviceId deviceId) { 167 + public synchronized RoleInfo getNodes(DeviceId deviceId) {
123 - List<NodeId> nodes = new ArrayList<>(); 168 + return new RoleInfo(masterMap.get(deviceId),
124 - nodes.addAll(backups); 169 + backups.getOrDefault(deviceId, ImmutableList.of()));
125 -
126 - return new RoleInfo(masterMap.get(deviceId), nodes);
127 } 170 }
128 171
129 @Override 172 @Override
...@@ -134,69 +177,97 @@ public class SimpleMastershipStore ...@@ -134,69 +177,97 @@ public class SimpleMastershipStore
134 ids.add(d.getKey()); 177 ids.add(d.getKey());
135 } 178 }
136 } 179 }
137 - return Collections.unmodifiableSet(ids); 180 + return ids;
138 } 181 }
139 182
140 @Override 183 @Override
141 - public MastershipRole requestRole(DeviceId deviceId) { 184 + public synchronized MastershipRole requestRole(DeviceId deviceId) {
142 //query+possible reelection 185 //query+possible reelection
143 - NodeId node = instance.id(); 186 + NodeId node = clusterService.getLocalNode().id();
144 MastershipRole role = getRole(node, deviceId); 187 MastershipRole role = getRole(node, deviceId);
145 188
146 switch (role) { 189 switch (role) {
147 case MASTER: 190 case MASTER:
148 - break; 191 + return MastershipRole.MASTER;
149 case STANDBY: 192 case STANDBY:
150 - synchronized (this) { 193 + if (getMaster(deviceId) == null) {
151 - //try to "re-elect", since we're really not distributed 194 + // no master => become master
152 - NodeId rel = reelect(node); 195 + masterMap.put(deviceId, node);
153 - if (rel == null) { 196 + incrementTerm(deviceId);
154 - masterMap.put(deviceId, node); 197 + // remove from backup list
155 - termMap.put(deviceId, new AtomicInteger(INIT)); 198 + removeFromBackups(deviceId, node);
156 - role = MastershipRole.MASTER; 199 + notifyDelegate(new MastershipEvent(MASTER_CHANGED, deviceId,
157 - } 200 + getNodes(deviceId)));
158 - backups.add(node); 201 + return MastershipRole.MASTER;
159 } 202 }
160 - break; 203 + return MastershipRole.STANDBY;
161 case NONE: 204 case NONE:
162 - //first to get to it, say we are master 205 + if (getMaster(deviceId) == null) {
163 - synchronized (this) { 206 + // no master => become master
164 masterMap.put(deviceId, node); 207 masterMap.put(deviceId, node);
165 - termMap.put(deviceId, new AtomicInteger(INIT)); 208 + incrementTerm(deviceId);
166 - backups.add(node); 209 + notifyDelegate(new MastershipEvent(MASTER_CHANGED, deviceId,
167 - role = MastershipRole.MASTER; 210 + getNodes(deviceId)));
211 + return MastershipRole.MASTER;
168 } 212 }
169 - break; 213 + // add to backup list
214 + if (addToBackup(deviceId, node)) {
215 + notifyDelegate(new MastershipEvent(BACKUPS_CHANGED, deviceId,
216 + getNodes(deviceId)));
217 + }
218 + return MastershipRole.STANDBY;
170 default: 219 default:
171 log.warn("unknown Mastership Role {}", role); 220 log.warn("unknown Mastership Role {}", role);
172 } 221 }
173 return role; 222 return role;
174 } 223 }
175 224
225 + // add to backup if not there already, silently ignores null node
226 + private synchronized boolean addToBackup(DeviceId deviceId, NodeId nodeId) {
227 + boolean modified = false;
228 + List<NodeId> stbys = backups.getOrDefault(deviceId, new ArrayList<>());
229 + if (nodeId != null && !stbys.contains(nodeId)) {
230 + stbys.add(nodeId);
231 + modified = true;
232 + }
233 + backups.put(deviceId, stbys);
234 + return modified;
235 + }
236 +
237 + private synchronized boolean removeFromBackups(DeviceId deviceId, NodeId node) {
238 + List<NodeId> stbys = backups.getOrDefault(deviceId, new ArrayList<>());
239 + boolean modified = stbys.remove(node);
240 + backups.put(deviceId, stbys);
241 + return modified;
242 + }
243 +
244 + private synchronized void incrementTerm(DeviceId deviceId) {
245 + AtomicInteger term = termMap.getOrDefault(deviceId, new AtomicInteger(NOTHING));
246 + term.incrementAndGet();
247 + termMap.put(deviceId, term);
248 + }
249 +
176 @Override 250 @Override
177 public MastershipRole getRole(NodeId nodeId, DeviceId deviceId) { 251 public MastershipRole getRole(NodeId nodeId, DeviceId deviceId) {
178 //just query 252 //just query
179 NodeId current = masterMap.get(deviceId); 253 NodeId current = masterMap.get(deviceId);
180 MastershipRole role; 254 MastershipRole role;
181 255
182 - if (current == null) { 256 + if (current != null && current.equals(nodeId)) {
183 - if (backups.contains(nodeId)) { 257 + return MastershipRole.MASTER;
184 - role = MastershipRole.STANDBY; 258 + }
185 - } else { 259 +
186 - role = MastershipRole.NONE; 260 + if (backups.getOrDefault(deviceId, Collections.emptyList()).contains(nodeId)) {
187 - } 261 + role = MastershipRole.STANDBY;
188 } else { 262 } else {
189 - if (current.equals(nodeId)) { 263 + role = MastershipRole.NONE;
190 - role = MastershipRole.MASTER;
191 - } else {
192 - role = MastershipRole.STANDBY;
193 - }
194 } 264 }
195 return role; 265 return role;
196 } 266 }
197 267
268 + // synchronized for atomic read
198 @Override 269 @Override
199 - public MastershipTerm getTermFor(DeviceId deviceId) { 270 + public synchronized MastershipTerm getTermFor(DeviceId deviceId) {
200 if ((termMap.get(deviceId) == null)) { 271 if ((termMap.get(deviceId) == null)) {
201 return MastershipTerm.of(masterMap.get(deviceId), NOTHING); 272 return MastershipTerm.of(masterMap.get(deviceId), NOTHING);
202 } 273 }
...@@ -205,72 +276,71 @@ public class SimpleMastershipStore ...@@ -205,72 +276,71 @@ public class SimpleMastershipStore
205 } 276 }
206 277
207 @Override 278 @Override
208 - public MastershipEvent setStandby(NodeId nodeId, DeviceId deviceId) { 279 + public synchronized MastershipEvent setStandby(NodeId nodeId, DeviceId deviceId) {
209 MastershipRole role = getRole(nodeId, deviceId); 280 MastershipRole role = getRole(nodeId, deviceId);
210 - synchronized (this) { 281 + switch (role) {
211 - switch (role) { 282 + case MASTER:
212 - case MASTER: 283 + NodeId backup = reelect(deviceId, nodeId);
213 - NodeId backup = reelect(nodeId); 284 + if (backup == null) {
214 - if (backup == null) { 285 + // no master alternative
215 - masterMap.remove(deviceId); 286 + masterMap.remove(deviceId);
216 - } else { 287 + // TODO: Should there be new event type for no MASTER?
217 - masterMap.put(deviceId, backup); 288 + return new MastershipEvent(MASTER_CHANGED, deviceId,
218 - termMap.get(deviceId).incrementAndGet(); 289 + getNodes(deviceId));
219 - return new MastershipEvent(MASTER_CHANGED, deviceId, 290 + } else {
220 - new RoleInfo(backup, Lists.newLinkedList(backups))); 291 + NodeId prevMaster = masterMap.put(deviceId, backup);
221 - } 292 + incrementTerm(deviceId);
222 - case STANDBY: 293 + addToBackup(deviceId, prevMaster);
223 - case NONE: 294 + return new MastershipEvent(MASTER_CHANGED, deviceId,
224 - if (!termMap.containsKey(deviceId)) { 295 + getNodes(deviceId));
225 - termMap.put(deviceId, new AtomicInteger(INIT)); 296 + }
226 - } 297 + case STANDBY:
227 - backups.add(nodeId); 298 + case NONE:
228 - break; 299 + boolean modified = addToBackup(deviceId, nodeId);
229 - default: 300 + if (modified) {
230 - log.warn("unknown Mastership Role {}", role); 301 + return new MastershipEvent(BACKUPS_CHANGED, deviceId,
302 + getNodes(deviceId));
231 } 303 }
304 + default:
305 + log.warn("unknown Mastership Role {}", role);
232 } 306 }
233 return null; 307 return null;
234 } 308 }
235 309
236 //dumbly selects next-available node that's not the current one 310 //dumbly selects next-available node that's not the current one
237 //emulate leader election 311 //emulate leader election
238 - private NodeId reelect(NodeId nodeId) { 312 + private synchronized NodeId reelect(DeviceId did, NodeId nodeId) {
313 + List<NodeId> stbys = backups.getOrDefault(did, Collections.emptyList());
239 NodeId backup = null; 314 NodeId backup = null;
240 - for (NodeId n : backups) { 315 + for (NodeId n : stbys) {
241 if (!n.equals(nodeId)) { 316 if (!n.equals(nodeId)) {
242 backup = n; 317 backup = n;
243 break; 318 break;
244 } 319 }
245 } 320 }
246 - backups.remove(backup); 321 + stbys.remove(backup);
247 return backup; 322 return backup;
248 } 323 }
249 324
250 @Override 325 @Override
251 - public MastershipEvent relinquishRole(NodeId nodeId, DeviceId deviceId) { 326 + public synchronized MastershipEvent relinquishRole(NodeId nodeId, DeviceId deviceId) {
252 MastershipRole role = getRole(nodeId, deviceId); 327 MastershipRole role = getRole(nodeId, deviceId);
253 - synchronized (this) { 328 + switch (role) {
254 - switch (role) { 329 + case MASTER:
255 - case MASTER: 330 + NodeId backup = reelect(deviceId, nodeId);
256 - NodeId backup = reelect(nodeId); 331 + masterMap.put(deviceId, backup);
257 - backups.remove(nodeId); 332 + incrementTerm(deviceId);
258 - if (backup == null) { 333 + return new MastershipEvent(MASTER_CHANGED, deviceId,
259 - masterMap.remove(deviceId); 334 + getNodes(deviceId));
260 - } else { 335 + case STANDBY:
261 - masterMap.put(deviceId, backup); 336 + if (removeFromBackups(deviceId, nodeId)) {
262 - termMap.get(deviceId).incrementAndGet(); 337 + return new MastershipEvent(BACKUPS_CHANGED, deviceId,
263 - return new MastershipEvent(MASTER_CHANGED, deviceId, 338 + getNodes(deviceId));
264 - new RoleInfo(backup, Lists.newLinkedList(backups)));
265 - }
266 - case STANDBY:
267 - backups.remove(nodeId);
268 - case NONE:
269 - default:
270 - log.warn("unknown Mastership Role {}", role);
271 } 339 }
340 + case NONE:
341 + default:
342 + log.warn("unknown Mastership Role {}", role);
272 } 343 }
273 return null; 344 return null;
274 } 345 }
275 -
276 } 346 }
......
...@@ -15,6 +15,8 @@ ...@@ -15,6 +15,8 @@
15 */ 15 */
16 package org.onlab.onos.store.trivial.impl; 16 package org.onlab.onos.store.trivial.impl;
17 17
18 +import java.util.ArrayList;
19 +import java.util.List;
18 import java.util.Set; 20 import java.util.Set;
19 import java.util.concurrent.atomic.AtomicInteger; 21 import java.util.concurrent.atomic.AtomicInteger;
20 22
...@@ -22,6 +24,7 @@ import org.junit.After; ...@@ -22,6 +24,7 @@ import org.junit.After;
22 import org.junit.Before; 24 import org.junit.Before;
23 import org.junit.Test; 25 import org.junit.Test;
24 import org.onlab.onos.cluster.NodeId; 26 import org.onlab.onos.cluster.NodeId;
27 +import org.onlab.onos.mastership.MastershipEvent;
25 import org.onlab.onos.mastership.MastershipTerm; 28 import org.onlab.onos.mastership.MastershipTerm;
26 import org.onlab.onos.net.DeviceId; 29 import org.onlab.onos.net.DeviceId;
27 30
...@@ -74,6 +77,7 @@ public class SimpleMastershipStoreTest { ...@@ -74,6 +77,7 @@ public class SimpleMastershipStoreTest {
74 assertEquals("wrong role", MASTER, sms.getRole(N2, DID3)); 77 assertEquals("wrong role", MASTER, sms.getRole(N2, DID3));
75 78
76 //N2 is master but N1 is only in backups set 79 //N2 is master but N1 is only in backups set
80 + put(DID4, N1, false, true);
77 put(DID4, N2, true, false); 81 put(DID4, N2, true, false);
78 assertEquals("wrong role", STANDBY, sms.getRole(N1, DID4)); 82 assertEquals("wrong role", STANDBY, sms.getRole(N1, DID4));
79 } 83 }
...@@ -127,12 +131,12 @@ public class SimpleMastershipStoreTest { ...@@ -127,12 +131,12 @@ public class SimpleMastershipStoreTest {
127 put(DID1, N1, false, false); 131 put(DID1, N1, false, false);
128 assertEquals("wrong role", MASTER, sms.requestRole(DID1)); 132 assertEquals("wrong role", MASTER, sms.requestRole(DID1));
129 133
130 - //STANDBY without backup - become MASTER 134 + //was STANDBY - become MASTER
131 put(DID2, N1, false, true); 135 put(DID2, N1, false, true);
132 assertEquals("wrong role", MASTER, sms.requestRole(DID2)); 136 assertEquals("wrong role", MASTER, sms.requestRole(DID2));
133 137
134 - //STANDBY with backup - stay STANDBY 138 + //other MASTER - stay STANDBY
135 - put(DID3, N2, false, true); 139 + put(DID3, N2, true, false);
136 assertEquals("wrong role", STANDBY, sms.requestRole(DID3)); 140 assertEquals("wrong role", STANDBY, sms.requestRole(DID3));
137 141
138 //local (N1) is MASTER - stay MASTER 142 //local (N1) is MASTER - stay MASTER
...@@ -145,30 +149,34 @@ public class SimpleMastershipStoreTest { ...@@ -145,30 +149,34 @@ public class SimpleMastershipStoreTest {
145 //NONE - record backup but take no other action 149 //NONE - record backup but take no other action
146 put(DID1, N1, false, false); 150 put(DID1, N1, false, false);
147 sms.setStandby(N1, DID1); 151 sms.setStandby(N1, DID1);
148 - assertTrue("not backed up", sms.backups.contains(N1)); 152 + assertTrue("not backed up", sms.backups.get(DID1).contains(N1));
149 - sms.termMap.clear(); 153 + int prev = sms.termMap.get(DID1).get();
150 sms.setStandby(N1, DID1); 154 sms.setStandby(N1, DID1);
151 - assertTrue("term not set", sms.termMap.containsKey(DID1)); 155 + assertEquals("term should not change", prev, sms.termMap.get(DID1).get());
152 156
153 //no backup, MASTER 157 //no backup, MASTER
154 - put(DID1, N1, true, true); 158 + put(DID1, N1, true, false);
155 - assertNull("wrong event", sms.setStandby(N1, DID1)); 159 + assertNull("expect no MASTER event", sms.setStandby(N1, DID1).roleInfo().master());
156 assertNull("wrong node", sms.masterMap.get(DID1)); 160 assertNull("wrong node", sms.masterMap.get(DID1));
157 161
158 //backup, switch 162 //backup, switch
159 sms.masterMap.clear(); 163 sms.masterMap.clear();
160 put(DID1, N1, true, true); 164 put(DID1, N1, true, true);
165 + put(DID1, N2, false, true);
161 put(DID2, N2, true, true); 166 put(DID2, N2, true, true);
162 - assertEquals("wrong event", MASTER_CHANGED, sms.setStandby(N1, DID1).type()); 167 + MastershipEvent event = sms.setStandby(N1, DID1);
168 + assertEquals("wrong event", MASTER_CHANGED, event.type());
169 + assertEquals("wrong master", N2, event.roleInfo().master());
163 } 170 }
164 171
165 //helper to populate master/backup structures 172 //helper to populate master/backup structures
166 - private void put(DeviceId dev, NodeId node, boolean store, boolean backup) { 173 + private void put(DeviceId dev, NodeId node, boolean master, boolean backup) {
167 - if (store) { 174 + if (master) {
168 sms.masterMap.put(dev, node); 175 sms.masterMap.put(dev, node);
169 - } 176 + } else if (backup) {
170 - if (backup) { 177 + List<NodeId> stbys = sms.backups.getOrDefault(dev, new ArrayList<>());
171 - sms.backups.add(node); 178 + stbys.add(node);
179 + sms.backups.put(dev, stbys);
172 } 180 }
173 sms.termMap.put(dev, new AtomicInteger()); 181 sms.termMap.put(dev, new AtomicInteger());
174 } 182 }
......