Committed by
Gerrit Code Review
Fixing issues on GUI server side. Adding command to balance mastership. Messing …
…with color scheme per feedback. Change-Id: I89fb52105f7e724167a417e033048e9c88f31eae
Showing
8 changed files
with
169 additions
and
20 deletions
1 | +/* | ||
2 | + * Copyright 2014 Open Networking Laboratory | ||
3 | + * | ||
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | + * you may not use this file except in compliance with the License. | ||
6 | + * You may obtain a copy of the License at | ||
7 | + * | ||
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | + * | ||
10 | + * Unless required by applicable law or agreed to in writing, software | ||
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | + * See the License for the specific language governing permissions and | ||
14 | + * limitations under the License. | ||
15 | + */ | ||
16 | +package org.onlab.onos.cli; | ||
17 | + | ||
18 | +import com.google.common.collect.HashMultimap; | ||
19 | +import com.google.common.collect.Multimap; | ||
20 | +import org.apache.karaf.shell.commands.Command; | ||
21 | +import org.onlab.onos.cluster.ClusterService; | ||
22 | +import org.onlab.onos.cluster.ControllerNode; | ||
23 | +import org.onlab.onos.mastership.MastershipAdminService; | ||
24 | +import org.onlab.onos.mastership.MastershipService; | ||
25 | +import org.onlab.onos.net.DeviceId; | ||
26 | +import org.onlab.onos.net.MastershipRole; | ||
27 | + | ||
28 | +import java.util.Collection; | ||
29 | +import java.util.Iterator; | ||
30 | +import java.util.List; | ||
31 | + | ||
32 | +import static com.google.common.collect.Lists.newArrayList; | ||
33 | + | ||
34 | +/** | ||
35 | + * Forces device mastership rebalancing. | ||
36 | + */ | ||
37 | +@Command(scope = "onos", name = "balance-masters", | ||
38 | + description = "Forces device mastership rebalancing") | ||
39 | +public class BalanceMastersCommand extends AbstractShellCommand { | ||
40 | + | ||
41 | + @Override | ||
42 | + protected void execute() { | ||
43 | + ClusterService service = get(ClusterService.class); | ||
44 | + MastershipService mastershipService = get(MastershipService.class); | ||
45 | + MastershipAdminService adminService = get(MastershipAdminService.class); | ||
46 | + | ||
47 | + List<ControllerNode> nodes = newArrayList(service.getNodes()); | ||
48 | + | ||
49 | + Multimap<ControllerNode, DeviceId> controllerDevices = HashMultimap.create(); | ||
50 | + | ||
51 | + // Create buckets reflecting current ownership. | ||
52 | + for (ControllerNode node : nodes) { | ||
53 | + controllerDevices.putAll(node, mastershipService.getDevicesOf(node.id())); | ||
54 | + } | ||
55 | + | ||
56 | + int bucketCount = nodes.size(); | ||
57 | + for (int i = 0; i < bucketCount / 2; i++) { | ||
58 | + // Iterate over the buckets and find the smallest and the largest. | ||
59 | + ControllerNode smallest = findSmallestBucket(controllerDevices); | ||
60 | + ControllerNode largest = findLargestBucket(controllerDevices); | ||
61 | + balanceBuckets(smallest, largest, controllerDevices, | ||
62 | + mastershipService, adminService); | ||
63 | + } | ||
64 | + } | ||
65 | + | ||
66 | + private ControllerNode findSmallestBucket(Multimap<ControllerNode, DeviceId> controllerDevices) { | ||
67 | + int minSize = Integer.MAX_VALUE; | ||
68 | + ControllerNode minNode = null; | ||
69 | + for (ControllerNode node : controllerDevices.keySet()) { | ||
70 | + int size = controllerDevices.get(node).size(); | ||
71 | + if (size < minSize) { | ||
72 | + minSize = size; | ||
73 | + minNode = node; | ||
74 | + } | ||
75 | + } | ||
76 | + return minNode; | ||
77 | + } | ||
78 | + | ||
79 | + private ControllerNode findLargestBucket(Multimap<ControllerNode, DeviceId> controllerDevices) { | ||
80 | + int maxSize = -1; | ||
81 | + ControllerNode maxNode = null; | ||
82 | + for (ControllerNode node : controllerDevices.keySet()) { | ||
83 | + int size = controllerDevices.get(node).size(); | ||
84 | + if (size >= maxSize) { | ||
85 | + maxSize = size; | ||
86 | + maxNode = node; | ||
87 | + } | ||
88 | + } | ||
89 | + return maxNode; | ||
90 | + } | ||
91 | + | ||
92 | + // FIXME: enhance to better handle cases where smallest cannot take any of the devices from largest | ||
93 | + | ||
94 | + private void balanceBuckets(ControllerNode smallest, ControllerNode largest, | ||
95 | + Multimap<ControllerNode, DeviceId> controllerDevices, | ||
96 | + MastershipService mastershipService, | ||
97 | + MastershipAdminService adminService) { | ||
98 | + Collection<DeviceId> minBucket = controllerDevices.get(smallest); | ||
99 | + Collection<DeviceId> maxBucket = controllerDevices.get(largest); | ||
100 | + | ||
101 | + int delta = (maxBucket.size() - minBucket.size()) / 2; | ||
102 | + | ||
103 | + print("Attempting to move %d nodes from %s to %s...", | ||
104 | + delta, largest.id(), smallest.id()); | ||
105 | + | ||
106 | + int i = 0; | ||
107 | + Iterator<DeviceId> it = maxBucket.iterator(); | ||
108 | + while (it.hasNext() && i < delta) { | ||
109 | + DeviceId deviceId = it.next(); | ||
110 | + | ||
111 | + // Check that the transfer can happen for the current element. | ||
112 | + if (mastershipService.getNodesFor(deviceId).backups().contains(smallest)) { | ||
113 | + print("Setting %s as the new master for %s", smallest.id(), deviceId); | ||
114 | + adminService.setRole(smallest.id(), deviceId, MastershipRole.MASTER); | ||
115 | + i++; | ||
116 | + } | ||
117 | + } | ||
118 | + | ||
119 | + controllerDevices.removeAll(smallest); | ||
120 | + } | ||
121 | + | ||
122 | +} |
... | @@ -61,6 +61,9 @@ | ... | @@ -61,6 +61,9 @@ |
61 | <action class="org.onlab.onos.cli.MastersListCommand"/> | 61 | <action class="org.onlab.onos.cli.MastersListCommand"/> |
62 | </command> | 62 | </command> |
63 | <command> | 63 | <command> |
64 | + <action class="org.onlab.onos.cli.BalanceMastersCommand"/> | ||
65 | + </command> | ||
66 | + <command> | ||
64 | <action class="org.onlab.onos.cli.ApplicationIdListCommand"/> | 67 | <action class="org.onlab.onos.cli.ApplicationIdListCommand"/> |
65 | </command> | 68 | </command> |
66 | 69 | ... | ... |
... | @@ -289,7 +289,7 @@ public abstract class TopologyViewMessages { | ... | @@ -289,7 +289,7 @@ public abstract class TopologyViewMessages { |
289 | } | 289 | } |
290 | 290 | ||
291 | // Produces a cluster instance message to the client. | 291 | // Produces a cluster instance message to the client. |
292 | - protected ObjectNode instanceMessage(ClusterEvent event) { | 292 | + protected ObjectNode instanceMessage(ClusterEvent event, String messageType) { |
293 | ControllerNode node = event.subject(); | 293 | ControllerNode node = event.subject(); |
294 | int switchCount = mastershipService.getDevicesOf(node.id()).size(); | 294 | int switchCount = mastershipService.getDevicesOf(node.id()).size(); |
295 | ObjectNode payload = mapper.createObjectNode() | 295 | ObjectNode payload = mapper.createObjectNode() |
... | @@ -307,8 +307,10 @@ public abstract class TopologyViewMessages { | ... | @@ -307,8 +307,10 @@ public abstract class TopologyViewMessages { |
307 | payload.set("labels", labels); | 307 | payload.set("labels", labels); |
308 | addMetaUi(node.id().toString(), payload); | 308 | addMetaUi(node.id().toString(), payload); |
309 | 309 | ||
310 | - String type = (event.type() == INSTANCE_ADDED) ? "addInstance" : | 310 | + String type = messageType != null ? messageType : |
311 | - ((event.type() == INSTANCE_REMOVED) ? "removeInstance" : "updateInstance"); | 311 | + ((event.type() == INSTANCE_ADDED) ? "addInstance" : |
312 | + ((event.type() == INSTANCE_REMOVED ? "removeInstance" : | ||
313 | + "updateInstance"))); | ||
312 | return envelope(type, 0, payload); | 314 | return envelope(type, 0, payload); |
313 | } | 315 | } |
314 | 316 | ... | ... |
... | @@ -24,6 +24,8 @@ import org.onlab.onos.cluster.ClusterEventListener; | ... | @@ -24,6 +24,8 @@ import org.onlab.onos.cluster.ClusterEventListener; |
24 | import org.onlab.onos.cluster.ControllerNode; | 24 | import org.onlab.onos.cluster.ControllerNode; |
25 | import org.onlab.onos.core.ApplicationId; | 25 | import org.onlab.onos.core.ApplicationId; |
26 | import org.onlab.onos.core.CoreService; | 26 | import org.onlab.onos.core.CoreService; |
27 | +import org.onlab.onos.mastership.MastershipEvent; | ||
28 | +import org.onlab.onos.mastership.MastershipListener; | ||
27 | import org.onlab.onos.net.ConnectPoint; | 29 | import org.onlab.onos.net.ConnectPoint; |
28 | import org.onlab.onos.net.Device; | 30 | import org.onlab.onos.net.Device; |
29 | import org.onlab.onos.net.Host; | 31 | import org.onlab.onos.net.Host; |
... | @@ -46,6 +48,7 @@ import org.onlab.onos.net.intent.MultiPointToSinglePointIntent; | ... | @@ -46,6 +48,7 @@ import org.onlab.onos.net.intent.MultiPointToSinglePointIntent; |
46 | import org.onlab.onos.net.link.LinkEvent; | 48 | import org.onlab.onos.net.link.LinkEvent; |
47 | import org.onlab.onos.net.link.LinkListener; | 49 | import org.onlab.onos.net.link.LinkListener; |
48 | import org.onlab.osgi.ServiceDirectory; | 50 | import org.onlab.osgi.ServiceDirectory; |
51 | +import org.onlab.packet.Ethernet; | ||
49 | 52 | ||
50 | import java.io.IOException; | 53 | import java.io.IOException; |
51 | import java.util.ArrayList; | 54 | import java.util.ArrayList; |
... | @@ -62,6 +65,7 @@ import static org.onlab.onos.cluster.ClusterEvent.Type.INSTANCE_ADDED; | ... | @@ -62,6 +65,7 @@ import static org.onlab.onos.cluster.ClusterEvent.Type.INSTANCE_ADDED; |
62 | import static org.onlab.onos.net.DeviceId.deviceId; | 65 | import static org.onlab.onos.net.DeviceId.deviceId; |
63 | import static org.onlab.onos.net.HostId.hostId; | 66 | import static org.onlab.onos.net.HostId.hostId; |
64 | import static org.onlab.onos.net.device.DeviceEvent.Type.DEVICE_ADDED; | 67 | import static org.onlab.onos.net.device.DeviceEvent.Type.DEVICE_ADDED; |
68 | +import static org.onlab.onos.net.device.DeviceEvent.Type.DEVICE_UPDATED; | ||
65 | import static org.onlab.onos.net.host.HostEvent.Type.HOST_ADDED; | 69 | import static org.onlab.onos.net.host.HostEvent.Type.HOST_ADDED; |
66 | import static org.onlab.onos.net.link.LinkEvent.Type.LINK_ADDED; | 70 | import static org.onlab.onos.net.link.LinkEvent.Type.LINK_ADDED; |
67 | 71 | ||
... | @@ -80,8 +84,8 @@ public class TopologyViewWebSocket | ... | @@ -80,8 +84,8 @@ public class TopologyViewWebSocket |
80 | 84 | ||
81 | private static final String APP_ID = "org.onlab.onos.gui"; | 85 | private static final String APP_ID = "org.onlab.onos.gui"; |
82 | 86 | ||
83 | - private static final long SUMMARY_FREQUENCY_SEC = 2000; | 87 | + private static final long SUMMARY_FREQUENCY_SEC = 3000; |
84 | - private static final long TRAFFIC_FREQUENCY_SEC = 1000; | 88 | + private static final long TRAFFIC_FREQUENCY_SEC = 1500; |
85 | 89 | ||
86 | private static final Comparator<? super ControllerNode> NODE_COMPARATOR = | 90 | private static final Comparator<? super ControllerNode> NODE_COMPARATOR = |
87 | new Comparator<ControllerNode>() { | 91 | new Comparator<ControllerNode>() { |
... | @@ -97,6 +101,7 @@ public class TopologyViewWebSocket | ... | @@ -97,6 +101,7 @@ public class TopologyViewWebSocket |
97 | private FrameConnection control; | 101 | private FrameConnection control; |
98 | 102 | ||
99 | private final ClusterEventListener clusterListener = new InternalClusterListener(); | 103 | private final ClusterEventListener clusterListener = new InternalClusterListener(); |
104 | + private final MastershipListener mastershipListener = new InternalMastershipListener(); | ||
100 | private final DeviceListener deviceListener = new InternalDeviceListener(); | 105 | private final DeviceListener deviceListener = new InternalDeviceListener(); |
101 | private final LinkListener linkListener = new InternalLinkListener(); | 106 | private final LinkListener linkListener = new InternalLinkListener(); |
102 | private final HostListener hostListener = new InternalHostListener(); | 107 | private final HostListener hostListener = new InternalHostListener(); |
... | @@ -164,7 +169,7 @@ public class TopologyViewWebSocket | ... | @@ -164,7 +169,7 @@ public class TopologyViewWebSocket |
164 | this.control = (FrameConnection) connection; | 169 | this.control = (FrameConnection) connection; |
165 | addListeners(); | 170 | addListeners(); |
166 | 171 | ||
167 | - sendAllInstances(); | 172 | + sendAllInstances(null); |
168 | sendAllDevices(); | 173 | sendAllDevices(); |
169 | sendAllLinks(); | 174 | sendAllLinks(); |
170 | sendAllHosts(); | 175 | sendAllHosts(); |
... | @@ -235,11 +240,12 @@ public class TopologyViewWebSocket | ... | @@ -235,11 +240,12 @@ public class TopologyViewWebSocket |
235 | } | 240 | } |
236 | 241 | ||
237 | // Sends all controller nodes to the client as node-added messages. | 242 | // Sends all controller nodes to the client as node-added messages. |
238 | - private void sendAllInstances() { | 243 | + private void sendAllInstances(String messageType) { |
239 | List<ControllerNode> nodes = new ArrayList<>(clusterService.getNodes()); | 244 | List<ControllerNode> nodes = new ArrayList<>(clusterService.getNodes()); |
240 | Collections.sort(nodes, NODE_COMPARATOR); | 245 | Collections.sort(nodes, NODE_COMPARATOR); |
241 | for (ControllerNode node : nodes) { | 246 | for (ControllerNode node : nodes) { |
242 | - sendMessage(instanceMessage(new ClusterEvent(INSTANCE_ADDED, node))); | 247 | + sendMessage(instanceMessage(new ClusterEvent(INSTANCE_ADDED, node), |
248 | + messageType)); | ||
243 | } | 249 | } |
244 | } | 250 | } |
245 | 251 | ||
... | @@ -307,13 +313,14 @@ public class TopologyViewWebSocket | ... | @@ -307,13 +313,14 @@ public class TopologyViewWebSocket |
307 | 313 | ||
308 | // FIXME: clearly, this is not enough | 314 | // FIXME: clearly, this is not enough |
309 | TrafficSelector selector = DefaultTrafficSelector.builder() | 315 | TrafficSelector selector = DefaultTrafficSelector.builder() |
316 | + .matchEthType(Ethernet.TYPE_IPV4) | ||
310 | .matchEthDst(dstHost.mac()).build(); | 317 | .matchEthDst(dstHost.mac()).build(); |
311 | TrafficTreatment treatment = DefaultTrafficTreatment.builder().build(); | 318 | TrafficTreatment treatment = DefaultTrafficTreatment.builder().build(); |
312 | 319 | ||
313 | MultiPointToSinglePointIntent intent = | 320 | MultiPointToSinglePointIntent intent = |
314 | new MultiPointToSinglePointIntent(appId, selector, treatment, | 321 | new MultiPointToSinglePointIntent(appId, selector, treatment, |
315 | ingressPoints, dstHost.location()); | 322 | ingressPoints, dstHost.location()); |
316 | - trafficEvent = event; | 323 | + startMonitoring(event); |
317 | intentService.submit(intent); | 324 | intentService.submit(intent); |
318 | } | 325 | } |
319 | 326 | ||
... | @@ -359,7 +366,6 @@ public class TopologyViewWebSocket | ... | @@ -359,7 +366,6 @@ public class TopologyViewWebSocket |
359 | 366 | ||
360 | // Subscribes for host traffic messages. | 367 | // Subscribes for host traffic messages. |
361 | private synchronized void requestAllTraffic(ObjectNode event) { | 368 | private synchronized void requestAllTraffic(ObjectNode event) { |
362 | - ObjectNode payload = payload(event); | ||
363 | long sid = startMonitoring(event); | 369 | long sid = startMonitoring(event); |
364 | sendMessage(trafficSummaryMessage(sid)); | 370 | sendMessage(trafficSummaryMessage(sid)); |
365 | } | 371 | } |
... | @@ -375,7 +381,6 @@ public class TopologyViewWebSocket | ... | @@ -375,7 +381,6 @@ public class TopologyViewWebSocket |
375 | 381 | ||
376 | // If there is a hover node, include it in the hosts and find intents. | 382 | // If there is a hover node, include it in the hosts and find intents. |
377 | String hover = string(payload, "hover"); | 383 | String hover = string(payload, "hover"); |
378 | - Set<Intent> hoverIntents; | ||
379 | if (!isNullOrEmpty(hover)) { | 384 | if (!isNullOrEmpty(hover)) { |
380 | addHover(hosts, devices, hover); | 385 | addHover(hosts, devices, hover); |
381 | } | 386 | } |
... | @@ -447,6 +452,7 @@ public class TopologyViewWebSocket | ... | @@ -447,6 +452,7 @@ public class TopologyViewWebSocket |
447 | // Adds all internal listeners. | 452 | // Adds all internal listeners. |
448 | private void addListeners() { | 453 | private void addListeners() { |
449 | clusterService.addListener(clusterListener); | 454 | clusterService.addListener(clusterListener); |
455 | + mastershipService.addListener(mastershipListener); | ||
450 | deviceService.addListener(deviceListener); | 456 | deviceService.addListener(deviceListener); |
451 | linkService.addListener(linkListener); | 457 | linkService.addListener(linkListener); |
452 | hostService.addListener(hostListener); | 458 | hostService.addListener(hostListener); |
... | @@ -458,6 +464,7 @@ public class TopologyViewWebSocket | ... | @@ -458,6 +464,7 @@ public class TopologyViewWebSocket |
458 | if (!listenersRemoved) { | 464 | if (!listenersRemoved) { |
459 | listenersRemoved = true; | 465 | listenersRemoved = true; |
460 | clusterService.removeListener(clusterListener); | 466 | clusterService.removeListener(clusterListener); |
467 | + mastershipService.removeListener(mastershipListener); | ||
461 | deviceService.removeListener(deviceListener); | 468 | deviceService.removeListener(deviceListener); |
462 | linkService.removeListener(linkListener); | 469 | linkService.removeListener(linkListener); |
463 | hostService.removeListener(hostListener); | 470 | hostService.removeListener(hostListener); |
... | @@ -469,7 +476,17 @@ public class TopologyViewWebSocket | ... | @@ -469,7 +476,17 @@ public class TopologyViewWebSocket |
469 | private class InternalClusterListener implements ClusterEventListener { | 476 | private class InternalClusterListener implements ClusterEventListener { |
470 | @Override | 477 | @Override |
471 | public void event(ClusterEvent event) { | 478 | public void event(ClusterEvent event) { |
472 | - sendMessage(instanceMessage(event)); | 479 | + sendMessage(instanceMessage(event, null)); |
480 | + } | ||
481 | + } | ||
482 | + | ||
483 | + // Mastership change listener | ||
484 | + private class InternalMastershipListener implements MastershipListener { | ||
485 | + @Override | ||
486 | + public void event(MastershipEvent event) { | ||
487 | + sendAllInstances("updateInstance"); | ||
488 | + Device device = deviceService.getDevice(event.subject()); | ||
489 | + sendMessage(deviceMessage(new DeviceEvent(DEVICE_UPDATED, device))); | ||
473 | } | 490 | } |
474 | } | 491 | } |
475 | 492 | ... | ... |
... | @@ -123,10 +123,15 @@ | ... | @@ -123,10 +123,15 @@ |
123 | // TODO: tune colors for light and dark themes | 123 | // TODO: tune colors for light and dark themes |
124 | 124 | ||
125 | // blue purple pink mustard cyan green red | 125 | // blue purple pink mustard cyan green red |
126 | - var lightNorm = ['#1f77b4', '#9467bd', '#e377c2', '#bcbd22', '#17becf', '#2ca02c', '#d62728'], | 126 | + //var lightNorm = ['#1f77b4', '#9467bd', '#e377c2', '#bcbd22', '#17becf', '#2ca02c', '#d62728'], |
127 | - lightMute = ['#aec7e8', '#c5b0d5', '#f7b6d2', '#dbdb8d', '#9edae5', '#98df8a', '#ff9896'], | 127 | + // lightMute = ['#aec7e8', '#c5b0d5', '#f7b6d2', '#dbdb8d', '#9edae5', '#98df8a', '#ff9896'], |
128 | - darkNorm = ['#1f77b4', '#9467bd', '#e377c2', '#bcbd22', '#17becf', '#2ca02c', '#d62728'], | 128 | + // darkNorm = ['#1f77b4', '#9467bd', '#e377c2', '#bcbd22', '#17becf', '#2ca02c', '#d62728'], |
129 | - darkMute = ['#aec7e8', '#c5b0d5', '#f7b6d2', '#dbdb8d', '#9edae5', '#98df8a', '#ff9896']; | 129 | + // darkMute = ['#aec7e8', '#c5b0d5', '#f7b6d2', '#dbdb8d', '#9edae5', '#98df8a', '#ff9896']; |
130 | + | ||
131 | + var lightNorm = ['#3F587F', '#77533D', '#C94E30', '#892D78', '#138C62', '#006D72', '#59AD00'], | ||
132 | + lightMute = ['#56657C', '#665F57', '#C68C7F', '#876E82', '#68897E', '#4E6F70', '#93AA7B'], | ||
133 | + darkNorm = ['#3F587F', '#77533D', '#C94E30', '#892D78', '#138C62', '#006D72', '#59AD00'], | ||
134 | + darkMute = ['#56657C', '#665F57', '#C68C7F', '#876E82', '#68897E', '#4E6F70', '#93AA7B']; | ||
130 | 135 | ||
131 | function cat7() { | 136 | function cat7() { |
132 | var colors = { | 137 | var colors = { | ... | ... |
... | @@ -356,7 +356,7 @@ svg .node.host circle { | ... | @@ -356,7 +356,7 @@ svg .node.host circle { |
356 | fill: #888; | 356 | fill: #888; |
357 | } | 357 | } |
358 | #topo-oibox .online svg text { | 358 | #topo-oibox .online svg text { |
359 | - fill: #000; | 359 | + fill: #eee; |
360 | } | 360 | } |
361 | 361 | ||
362 | #topo-oibox svg text.instTitle { | 362 | #topo-oibox svg text.instTitle { | ... | ... |
... | @@ -1088,8 +1088,8 @@ | ... | @@ -1088,8 +1088,8 @@ |
1088 | 1088 | ||
1089 | svg.append('rect').attr(rectAttr); | 1089 | svg.append('rect').attr(rectAttr); |
1090 | 1090 | ||
1091 | - appendGlyph(svg, c.nodeOx, c.nodeOy, c.nodeDim, '#node'); | 1091 | + //appendGlyph(svg, c.nodeOx, c.nodeOy, c.nodeDim, '#node'); |
1092 | - appendBadge(svg, c.birdOx, c.birdOy, c.birdDim, '#bird'); | 1092 | + appendBadge(svg, 14, 14, 28, '#bird'); |
1093 | 1093 | ||
1094 | if (d.uiAttached) { | 1094 | if (d.uiAttached) { |
1095 | attachUiBadge(svg); | 1095 | attachUiBadge(svg); | ... | ... |
-
Please register or login to post a comment