홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Thomas Vachuska
1e68bdd9 1 parent 6e18fe39

Added ability to force mastership re-balancing between instances from the GUI. 

Change-Id: I98e56deb3e2b00df630ed85b596c8e35b3d6efab
Inline Side-by-side
Showing 8 changed files with 111 additions and 95 deletions
...@@ -15,23 +15,8 @@ ...@@ -15,23 +15,8 @@
15 */ 15 */
16 package org.onlab.onos.cli; 16 package org.onlab.onos.cli;
17 17
18 -import com.google.common.collect.HashMultimap;
19 -import com.google.common.collect.Multimap;
20 import org.apache.karaf.shell.commands.Command; 18 import org.apache.karaf.shell.commands.Command;
21 -import org.onlab.onos.cluster.ClusterService;
22 -import org.onlab.onos.cluster.ControllerNode;
23 import org.onlab.onos.mastership.MastershipAdminService; 19 import org.onlab.onos.mastership.MastershipAdminService;
24 -import org.onlab.onos.mastership.MastershipService;
25 -import org.onlab.onos.net.DeviceId;
26 -import org.onlab.onos.net.device.DeviceService;
27 -
28 -import java.util.Collection;
29 -import java.util.Iterator;
30 -import java.util.List;
31 -import java.util.Set;
32 -
33 -import static com.google.common.collect.Lists.newArrayList;
34 -import static org.onlab.onos.net.MastershipRole.MASTER;
35 20
36 /** 21 /**
37 * Forces device mastership rebalancing. 22 * Forces device mastership rebalancing.
...@@ -42,72 +27,7 @@ public class BalanceMastersCommand extends AbstractShellCommand { ...@@ -42,72 +27,7 @@ public class BalanceMastersCommand extends AbstractShellCommand {
42 27
43 @Override 28 @Override
44 protected void execute() { 29 protected void execute() {
45 - ClusterService service = get(ClusterService.class); 30 + get(MastershipAdminService.class).balanceRoles();
46 - MastershipService mastershipService = get(MastershipService.class);
47 - MastershipAdminService adminService = get(MastershipAdminService.class);
48 -
49 - List<ControllerNode> nodes = newArrayList(service.getNodes());
50 -
51 - Multimap<ControllerNode, DeviceId> controllerDevices = HashMultimap.create();
52 -
53 - // Create buckets reflecting current ownership.
54 - for (ControllerNode node : nodes) {
55 - Set<DeviceId> devicesOf = mastershipService.getDevicesOf(node.id());
56 - controllerDevices.putAll(node, devicesOf);
57 - print("Node %s has %d devices.", node.id(), devicesOf.size());
58 - }
59 -
60 - int rounds = nodes.size();
61 - for (int i = 0; i < rounds; i++) {
62 - // Iterate over the buckets and find the smallest and the largest.
63 - ControllerNode smallest = findBucket(true, nodes, controllerDevices);
64 - ControllerNode largest = findBucket(false, nodes, controllerDevices);
65 - balanceBuckets(smallest, largest, controllerDevices, adminService);
66 - }
67 - }
68 -
69 - private ControllerNode findBucket(boolean min, Collection<ControllerNode> nodes,
70 - Multimap<ControllerNode, DeviceId> controllerDevices) {
71 - int xSize = min ? Integer.MAX_VALUE : -1;
72 - ControllerNode xNode = null;
73 - for (ControllerNode node : nodes) {
74 - int size = controllerDevices.get(node).size();
75 - if ((min && size < xSize) || (!min && size > xSize)) {
76 - xSize = size;
77 - xNode = node;
78 - }
79 - }
80 - return xNode;
81 - }
82 -
83 - // FIXME: enhance to better handle cases where smallest cannot take any of the devices from largest
84 -
85 - private void balanceBuckets(ControllerNode smallest, ControllerNode largest,
86 - Multimap<ControllerNode, DeviceId> controllerDevices,
87 - MastershipAdminService adminService) {
88 - Collection<DeviceId> minBucket = controllerDevices.get(smallest);
89 - Collection<DeviceId> maxBucket = controllerDevices.get(largest);
90 - int bucketCount = controllerDevices.keySet().size();
91 - int deviceCount = get(DeviceService.class).getDeviceCount();
92 -
93 - int delta = (maxBucket.size() - minBucket.size()) / 2;
94 - delta = Math.min(deviceCount / bucketCount, delta);
95 -
96 - if (delta > 0) {
97 - print("Attempting to move %d nodes from %s to %s...",
98 - delta, largest.id(), smallest.id());
99 -
100 - int i = 0;
101 - Iterator<DeviceId> it = maxBucket.iterator();
102 - while (it.hasNext() && i < delta) {
103 - DeviceId deviceId = it.next();
104 - print("Setting %s as the master for %s", smallest.id(), deviceId);
105 - adminService.setRole(smallest.id(), deviceId, MASTER);
106 - controllerDevices.put(smallest, deviceId);
107 - it.remove();
108 - i++;
109 - }
110 - }
111 } 31 }
112 32
113 } 33 }
......
...@@ -33,4 +33,10 @@ public interface MastershipAdminService { ...@@ -33,4 +33,10 @@ public interface MastershipAdminService {
33 */ 33 */
34 void setRole(NodeId instance, DeviceId deviceId, MastershipRole role); 34 void setRole(NodeId instance, DeviceId deviceId, MastershipRole role);
35 35
36 + /**
37 + * Balances the mastership to be shared as evenly as possibly by all
38 + * online instances.
39 + */
40 + void balanceRoles();
41 +
36 } 42 }
......
...@@ -15,13 +15,10 @@ ...@@ -15,13 +15,10 @@
15 */ 15 */
16 package org.onlab.onos.cluster.impl; 16 package org.onlab.onos.cluster.impl;
17 17
18 -import static com.google.common.base.Preconditions.checkNotNull; 18 +import com.codahale.metrics.Timer;
19 -import static org.slf4j.LoggerFactory.getLogger; 19 +import com.codahale.metrics.Timer.Context;
20 -import static org.onlab.metrics.MetricsUtil.*; 20 +import com.google.common.collect.HashMultimap;
21 - 21 +import com.google.common.collect.Multimap;
22 -import java.util.Set;
23 -import java.util.concurrent.atomic.AtomicInteger;
24 -
25 import org.apache.felix.scr.annotations.Activate; 22 import org.apache.felix.scr.annotations.Activate;
26 import org.apache.felix.scr.annotations.Component; 23 import org.apache.felix.scr.annotations.Component;
27 import org.apache.felix.scr.annotations.Deactivate; 24 import org.apache.felix.scr.annotations.Deactivate;
...@@ -50,8 +47,18 @@ import org.onlab.onos.net.DeviceId; ...@@ -50,8 +47,18 @@ import org.onlab.onos.net.DeviceId;
50 import org.onlab.onos.net.MastershipRole; 47 import org.onlab.onos.net.MastershipRole;
51 import org.slf4j.Logger; 48 import org.slf4j.Logger;
52 49
53 -import com.codahale.metrics.Timer; 50 +import java.util.Collection;
54 -import com.codahale.metrics.Timer.Context; 51 +import java.util.Iterator;
52 +import java.util.List;
53 +import java.util.Set;
54 +import java.util.concurrent.atomic.AtomicInteger;
55 +
56 +import static com.google.common.base.Preconditions.checkNotNull;
57 +import static com.google.common.collect.Lists.newArrayList;
58 +import static org.onlab.metrics.MetricsUtil.startTimer;
59 +import static org.onlab.metrics.MetricsUtil.stopTimer;
60 +import static org.onlab.onos.net.MastershipRole.MASTER;
61 +import static org.slf4j.LoggerFactory.getLogger;
55 62
56 @Component(immediate = true) 63 @Component(immediate = true)
57 @Service 64 @Service
...@@ -198,6 +205,71 @@ public class MastershipManager ...@@ -198,6 +205,71 @@ public class MastershipManager
198 return metricsService; 205 return metricsService;
199 } 206 }
200 207
208 + @Override
209 + public void balanceRoles() {
210 + List<ControllerNode> nodes = newArrayList(clusterService.getNodes());
211 + Multimap<ControllerNode, DeviceId> controllerDevices = HashMultimap.create();
212 + int deviceCount = 0;
213 +
214 + // Create buckets reflecting current ownership.
215 + for (ControllerNode node : nodes) {
216 + Set<DeviceId> devicesOf = getDevicesOf(node.id());
217 + deviceCount += devicesOf.size();
218 + controllerDevices.putAll(node, devicesOf);
219 + log.info("Node {} has {} devices.", node.id(), devicesOf.size());
220 + }
221 +
222 + int rounds = nodes.size();
223 + for (int i = 0; i < rounds; i++) {
224 + // Iterate over the buckets and find the smallest and the largest.
225 + ControllerNode smallest = findBucket(true, nodes, controllerDevices);
226 + ControllerNode largest = findBucket(false, nodes, controllerDevices);
227 + balanceBuckets(smallest, largest, controllerDevices, deviceCount);
228 + }
229 + }
230 +
231 + private ControllerNode findBucket(boolean min, Collection<ControllerNode> nodes,
232 + Multimap<ControllerNode, DeviceId> controllerDevices) {
233 + int xSize = min ? Integer.MAX_VALUE : -1;
234 + ControllerNode xNode = null;
235 + for (ControllerNode node : nodes) {
236 + int size = controllerDevices.get(node).size();
237 + if ((min && size < xSize) || (!min && size > xSize)) {
238 + xSize = size;
239 + xNode = node;
240 + }
241 + }
242 + return xNode;
243 + }
244 +
245 + private void balanceBuckets(ControllerNode smallest, ControllerNode largest,
246 + Multimap<ControllerNode, DeviceId> controllerDevices,
247 + int deviceCount) {
248 + Collection<DeviceId> minBucket = controllerDevices.get(smallest);
249 + Collection<DeviceId> maxBucket = controllerDevices.get(largest);
250 + int bucketCount = controllerDevices.keySet().size();
251 +
252 + int delta = (maxBucket.size() - minBucket.size()) / 2;
253 + delta = Math.min(deviceCount / bucketCount, delta);
254 +
255 + if (delta > 0) {
256 + log.info("Attempting to move {} nodes from {} to {}...", delta,
257 + largest.id(), smallest.id());
258 +
259 + int i = 0;
260 + Iterator<DeviceId> it = maxBucket.iterator();
261 + while (it.hasNext() && i < delta) {
262 + DeviceId deviceId = it.next();
263 + log.info("Setting {} as the master for {}", smallest.id(), deviceId);
264 + setRole(smallest.id(), deviceId, MASTER);
265 + controllerDevices.put(smallest, deviceId);
266 + it.remove();
267 + i++;
268 + }
269 + }
270 + }
271 +
272 +
201 // Posts the specified event to the local event dispatcher. 273 // Posts the specified event to the local event dispatcher.
202 private void post(MastershipEvent event) { 274 private void post(MastershipEvent event) {
203 if (event != null && eventDispatcher != null) { 275 if (event != null && eventDispatcher != null) {
......
...@@ -27,6 +27,7 @@ import org.onlab.onos.core.CoreService; ...@@ -27,6 +27,7 @@ import org.onlab.onos.core.CoreService;
27 import org.onlab.onos.event.AbstractEventAccumulator; 27 import org.onlab.onos.event.AbstractEventAccumulator;
28 import org.onlab.onos.event.Event; 28 import org.onlab.onos.event.Event;
29 import org.onlab.onos.event.EventAccumulator; 29 import org.onlab.onos.event.EventAccumulator;
30 +import org.onlab.onos.mastership.MastershipAdminService;
30 import org.onlab.onos.mastership.MastershipEvent; 31 import org.onlab.onos.mastership.MastershipEvent;
31 import org.onlab.onos.mastership.MastershipListener; 32 import org.onlab.onos.mastership.MastershipListener;
32 import org.onlab.onos.net.ConnectPoint; 33 import org.onlab.onos.net.ConnectPoint;
...@@ -233,6 +234,9 @@ public class TopologyViewWebSocket ...@@ -233,6 +234,9 @@ public class TopologyViewWebSocket
233 requestSummary(event); 234 requestSummary(event);
234 } else if (type.equals("cancelSummary")) { 235 } else if (type.equals("cancelSummary")) {
235 cancelSummary(event); 236 cancelSummary(event);
237 +
238 + } else if (type.equals("equalizeMasters")) {
239 + equalizeMasters(event);
236 } 240 }
237 } 241 }
238 242
...@@ -449,6 +453,12 @@ public class TopologyViewWebSocket ...@@ -449,6 +453,12 @@ public class TopologyViewWebSocket
449 } 453 }
450 454
451 455
456 + // Forces mastership role rebalancing.
457 + private void equalizeMasters(ObjectNode event) {
458 + directory.get(MastershipAdminService.class).balanceRoles();
459 + }
460 +
461 +
452 // Adds all internal listeners. 462 // Adds all internal listeners.
453 private void addListeners() { 463 private void addListeners() {
454 clusterService.addListener(clusterListener); 464 clusterService.addListener(clusterListener);
......
...@@ -140,12 +140,12 @@ ...@@ -140,12 +140,12 @@
140 // TODO: tune colors for light and dark themes 140 // TODO: tune colors for light and dark themes
141 // Note: These colors look good on the white background. Still, need to tune for dark. 141 // Note: These colors look good on the white background. Still, need to tune for dark.
142 142
143 - // blue brown purple sea green brick red dark teal lime 143 + // blue brown brick red sea green purple dark teal lime
144 - var lightNorm = ['#3E5780', '#78533B', '#8A2979', '#018D61', '#CB4D28', '#006D73', '#56AF00'], 144 + var lightNorm = ['#3E5780', '#78533B', '#CB4D28', '#018D61', '#8A2979', '#006D73', '#56AF00'],
145 - lightMute = ['#A8B8CC', '#CCB3A8', '#D19FCE', '#96D6BF', '#FFC2BD', '#8FCCCA', '#CAEAA4'], 145 + lightMute = ['#A8B8CC', '#CCB3A8', '#FFC2BD', '#96D6BF', '#D19FCE', '#8FCCCA', '#CAEAA4'],
146 146
147 - darkNorm = ['#3E5780', '#78533B', '#8A2979', '#018D61', '#CB4D28', '#006D73', '#56AF00'], 147 + darkNorm = ['#3E5780', '#78533B', '#CB4D28', '#018D61', '#8A2979', '#006D73', '#56AF00'],
148 - darkMute = ['#A8B8CC', '#CCB3A8', '#D19FCE', '#96D6BF', '#FFC2BD', '#8FCCCA', '#CAEAA4']; 148 + darkMute = ['#A8B8CC', '#CCB3A8', '#FFC2BD', '#96D6BF', '#D19FCE', '#8FCCCA', '#CAEAA4'];
149 149
150 function cat7() { 150 function cat7() {
151 var colors = { 151 var colors = {
......
...@@ -97,6 +97,7 @@ ...@@ -97,6 +97,7 @@
97 case 187: return 'equals'; 97 case 187: return 'equals';
98 case 189: return 'dash'; 98 case 189: return 'dash';
99 case 191: return 'slash'; 99 case 191: return 'slash';
100 + case 192: return 'backQuote';
100 default: 101 default:
101 if ((code >= 48 && code <= 57) || 102 if ((code >= 48 && code <= 57) ||
102 (code >= 65 && code <= 90)) { 103 (code >= 65 && code <= 90)) {
......
...@@ -55,6 +55,7 @@ ...@@ -55,6 +55,7 @@
55 equals: '=', 55 equals: '=',
56 dash: '-', 56 dash: '-',
57 slash: '/', 57 slash: '/',
58 + backQuote: '`',
58 leftArrow: 'L-arrow', 59 leftArrow: 'L-arrow',
59 upArrow: 'U-arrow', 60 upArrow: 'U-arrow',
60 rightArrow: 'R-arrow', 61 rightArrow: 'R-arrow',
......
...@@ -140,6 +140,7 @@ ...@@ -140,6 +140,7 @@
140 equals: injectStartupEvents, 140 equals: injectStartupEvents,
141 dash: injectTestEvent, 141 dash: injectTestEvent,
142 142
143 + E: [equalizeMasters, 'Equalize mastership roles'],
143 O: [toggleSummary, 'Toggle ONOS summary pane'], 144 O: [toggleSummary, 'Toggle ONOS summary pane'],
144 I: [toggleInstances, 'Toggle ONOS instances pane'], 145 I: [toggleInstances, 'Toggle ONOS instances pane'],
145 D: [toggleDetails, 'Disable / enable details pane'], 146 D: [toggleDetails, 'Disable / enable details pane'],
...@@ -926,6 +927,11 @@ ...@@ -926,6 +927,11 @@
926 updateDeviceColors(); 927 updateDeviceColors();
927 } 928 }
928 929
930 + function equalizeMasters() {
931 + flash('Equalizing master roles');
932 + sendMessage('equalizeMasters');
933 + }
934 +
929 function toggleSummary() { 935 function toggleSummary() {
930 if (!summaryPane.isVisible()) { 936 if (!summaryPane.isVisible()) {
931 requestSummary(); 937 requestSummary();
......