홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Thomas Vachuska
1e68bdd9 1 parent 6e18fe39

Added ability to force mastership re-balancing between instances from the GUI. 

Change-Id: I98e56deb3e2b00df630ed85b596c8e35b3d6efab
Inline Side-by-side
Showing 8 changed files with 111 additions and 95 deletions
......@@ -15,23 +15,8 @@
*/
package org.onlab.onos.cli;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import org.apache.karaf.shell.commands.Command;
import org.onlab.onos.cluster.ClusterService;
import org.onlab.onos.cluster.ControllerNode;
import org.onlab.onos.mastership.MastershipAdminService;
import org.onlab.onos.mastership.MastershipService;
import org.onlab.onos.net.DeviceId;
import org.onlab.onos.net.device.DeviceService;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import static com.google.common.collect.Lists.newArrayList;
import static org.onlab.onos.net.MastershipRole.MASTER;
/**
* Forces device mastership rebalancing.
......@@ -42,72 +27,7 @@ public class BalanceMastersCommand extends AbstractShellCommand {
@Override
protected void execute() {
ClusterService service = get(ClusterService.class);
MastershipService mastershipService = get(MastershipService.class);
MastershipAdminService adminService = get(MastershipAdminService.class);
List<ControllerNode> nodes = newArrayList(service.getNodes());
Multimap<ControllerNode, DeviceId> controllerDevices = HashMultimap.create();
// Create buckets reflecting current ownership.
for (ControllerNode node : nodes) {
Set<DeviceId> devicesOf = mastershipService.getDevicesOf(node.id());
controllerDevices.putAll(node, devicesOf);
print("Node %s has %d devices.", node.id(), devicesOf.size());
}
int rounds = nodes.size();
for (int i = 0; i < rounds; i++) {
// Iterate over the buckets and find the smallest and the largest.
ControllerNode smallest = findBucket(true, nodes, controllerDevices);
ControllerNode largest = findBucket(false, nodes, controllerDevices);
balanceBuckets(smallest, largest, controllerDevices, adminService);
}
}
private ControllerNode findBucket(boolean min, Collection<ControllerNode> nodes,
Multimap<ControllerNode, DeviceId> controllerDevices) {
int xSize = min ? Integer.MAX_VALUE : -1;
ControllerNode xNode = null;
for (ControllerNode node : nodes) {
int size = controllerDevices.get(node).size();
if ((min && size < xSize) || (!min && size > xSize)) {
xSize = size;
xNode = node;
}
}
return xNode;
}
// FIXME: enhance to better handle cases where smallest cannot take any of the devices from largest
private void balanceBuckets(ControllerNode smallest, ControllerNode largest,
Multimap<ControllerNode, DeviceId> controllerDevices,
MastershipAdminService adminService) {
Collection<DeviceId> minBucket = controllerDevices.get(smallest);
Collection<DeviceId> maxBucket = controllerDevices.get(largest);
int bucketCount = controllerDevices.keySet().size();
int deviceCount = get(DeviceService.class).getDeviceCount();
int delta = (maxBucket.size() - minBucket.size()) / 2;
delta = Math.min(deviceCount / bucketCount, delta);
if (delta > 0) {
print("Attempting to move %d nodes from %s to %s...",
delta, largest.id(), smallest.id());
int i = 0;
Iterator<DeviceId> it = maxBucket.iterator();
while (it.hasNext() && i < delta) {
DeviceId deviceId = it.next();
print("Setting %s as the master for %s", smallest.id(), deviceId);
adminService.setRole(smallest.id(), deviceId, MASTER);
controllerDevices.put(smallest, deviceId);
it.remove();
i++;
}
}
get(MastershipAdminService.class).balanceRoles();
}
}
......
......@@ -33,4 +33,10 @@ public interface MastershipAdminService {
*/
void setRole(NodeId instance, DeviceId deviceId, MastershipRole role);
/**
* Balances the mastership to be shared as evenly as possibly by all
* online instances.
*/
void balanceRoles();
}
......
......@@ -15,13 +15,10 @@
*/
package org.onlab.onos.cluster.impl;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.slf4j.LoggerFactory.getLogger;
import static org.onlab.metrics.MetricsUtil.*;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import com.codahale.metrics.Timer;
import com.codahale.metrics.Timer.Context;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
......@@ -50,8 +47,18 @@ import org.onlab.onos.net.DeviceId;
import org.onlab.onos.net.MastershipRole;
import org.slf4j.Logger;
import com.codahale.metrics.Timer;
import com.codahale.metrics.Timer.Context;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Lists.newArrayList;
import static org.onlab.metrics.MetricsUtil.startTimer;
import static org.onlab.metrics.MetricsUtil.stopTimer;
import static org.onlab.onos.net.MastershipRole.MASTER;
import static org.slf4j.LoggerFactory.getLogger;
@Component(immediate = true)
@Service
......@@ -198,6 +205,71 @@ public class MastershipManager
return metricsService;
}
@Override
public void balanceRoles() {
List<ControllerNode> nodes = newArrayList(clusterService.getNodes());
Multimap<ControllerNode, DeviceId> controllerDevices = HashMultimap.create();
int deviceCount = 0;
// Create buckets reflecting current ownership.
for (ControllerNode node : nodes) {
Set<DeviceId> devicesOf = getDevicesOf(node.id());
deviceCount += devicesOf.size();
controllerDevices.putAll(node, devicesOf);
log.info("Node {} has {} devices.", node.id(), devicesOf.size());
}
int rounds = nodes.size();
for (int i = 0; i < rounds; i++) {
// Iterate over the buckets and find the smallest and the largest.
ControllerNode smallest = findBucket(true, nodes, controllerDevices);
ControllerNode largest = findBucket(false, nodes, controllerDevices);
balanceBuckets(smallest, largest, controllerDevices, deviceCount);
}
}
private ControllerNode findBucket(boolean min, Collection<ControllerNode> nodes,
Multimap<ControllerNode, DeviceId> controllerDevices) {
int xSize = min ? Integer.MAX_VALUE : -1;
ControllerNode xNode = null;
for (ControllerNode node : nodes) {
int size = controllerDevices.get(node).size();
if ((min && size < xSize) || (!min && size > xSize)) {
xSize = size;
xNode = node;
}
}
return xNode;
}
private void balanceBuckets(ControllerNode smallest, ControllerNode largest,
Multimap<ControllerNode, DeviceId> controllerDevices,
int deviceCount) {
Collection<DeviceId> minBucket = controllerDevices.get(smallest);
Collection<DeviceId> maxBucket = controllerDevices.get(largest);
int bucketCount = controllerDevices.keySet().size();
int delta = (maxBucket.size() - minBucket.size()) / 2;
delta = Math.min(deviceCount / bucketCount, delta);
if (delta > 0) {
log.info("Attempting to move {} nodes from {} to {}...", delta,
largest.id(), smallest.id());
int i = 0;
Iterator<DeviceId> it = maxBucket.iterator();
while (it.hasNext() && i < delta) {
DeviceId deviceId = it.next();
log.info("Setting {} as the master for {}", smallest.id(), deviceId);
setRole(smallest.id(), deviceId, MASTER);
controllerDevices.put(smallest, deviceId);
it.remove();
i++;
}
}
}
// Posts the specified event to the local event dispatcher.
private void post(MastershipEvent event) {
if (event != null && eventDispatcher != null) {
......
......@@ -27,6 +27,7 @@ import org.onlab.onos.core.CoreService;
import org.onlab.onos.event.AbstractEventAccumulator;
import org.onlab.onos.event.Event;
import org.onlab.onos.event.EventAccumulator;
import org.onlab.onos.mastership.MastershipAdminService;
import org.onlab.onos.mastership.MastershipEvent;
import org.onlab.onos.mastership.MastershipListener;
import org.onlab.onos.net.ConnectPoint;
......@@ -233,6 +234,9 @@ public class TopologyViewWebSocket
requestSummary(event);
} else if (type.equals("cancelSummary")) {
cancelSummary(event);
} else if (type.equals("equalizeMasters")) {
equalizeMasters(event);
}
}
......@@ -449,6 +453,12 @@ public class TopologyViewWebSocket
}
// Forces mastership role rebalancing.
private void equalizeMasters(ObjectNode event) {
directory.get(MastershipAdminService.class).balanceRoles();
}
// Adds all internal listeners.
private void addListeners() {
clusterService.addListener(clusterListener);
......
......@@ -140,12 +140,12 @@
// TODO: tune colors for light and dark themes
// Note: These colors look good on the white background. Still, need to tune for dark.
// blue brown purple sea green brick red dark teal lime
var lightNorm = ['#3E5780', '#78533B', '#8A2979', '#018D61', '#CB4D28', '#006D73', '#56AF00'],
lightMute = ['#A8B8CC', '#CCB3A8', '#D19FCE', '#96D6BF', '#FFC2BD', '#8FCCCA', '#CAEAA4'],
// blue brown brick red sea green purple dark teal lime
var lightNorm = ['#3E5780', '#78533B', '#CB4D28', '#018D61', '#8A2979', '#006D73', '#56AF00'],
lightMute = ['#A8B8CC', '#CCB3A8', '#FFC2BD', '#96D6BF', '#D19FCE', '#8FCCCA', '#CAEAA4'],
darkNorm = ['#3E5780', '#78533B', '#8A2979', '#018D61', '#CB4D28', '#006D73', '#56AF00'],
darkMute = ['#A8B8CC', '#CCB3A8', '#D19FCE', '#96D6BF', '#FFC2BD', '#8FCCCA', '#CAEAA4'];
darkNorm = ['#3E5780', '#78533B', '#CB4D28', '#018D61', '#8A2979', '#006D73', '#56AF00'],
darkMute = ['#A8B8CC', '#CCB3A8', '#FFC2BD', '#96D6BF', '#D19FCE', '#8FCCCA', '#CAEAA4'];
function cat7() {
var colors = {
......
......@@ -97,6 +97,7 @@
case 187: return 'equals';
case 189: return 'dash';
case 191: return 'slash';
case 192: return 'backQuote';
default:
if ((code >= 48 && code <= 57) ||
(code >= 65 && code <= 90)) {
......
......@@ -55,6 +55,7 @@
equals: '=',
dash: '-',
slash: '/',
backQuote: '`',
leftArrow: 'L-arrow',
upArrow: 'U-arrow',
rightArrow: 'R-arrow',
......
......@@ -140,6 +140,7 @@
equals: injectStartupEvents,
dash: injectTestEvent,
E: [equalizeMasters, 'Equalize mastership roles'],
O: [toggleSummary, 'Toggle ONOS summary pane'],
I: [toggleInstances, 'Toggle ONOS instances pane'],
D: [toggleDetails, 'Disable / enable details pane'],
......@@ -926,6 +927,11 @@
updateDeviceColors();
}
function equalizeMasters() {
flash('Equalizing master roles');
sendMessage('equalizeMasters');
}
function toggleSummary() {
if (!summaryPane.isVisible()) {
requestSummary();
......