Committed by
Ray Milkey
Adding device and host tracking for intents (ONOS-1356)
Also, this should fix ONOS-1184 (intents submitted before hosts detected). Change-Id: I47a503c18dc728912132eb2e2fcc160d47e518eb
Showing
5 changed files
with
188 additions
and
11 deletions
... | @@ -18,5 +18,5 @@ package org.onosproject.net; | ... | @@ -18,5 +18,5 @@ package org.onosproject.net; |
18 | /** | 18 | /** |
19 | * Immutable representation of a network element identity. | 19 | * Immutable representation of a network element identity. |
20 | */ | 20 | */ |
21 | -public abstract class ElementId { | 21 | +public abstract class ElementId implements NetworkResource { |
22 | } | 22 | } | ... | ... |
... | @@ -15,15 +15,14 @@ | ... | @@ -15,15 +15,14 @@ |
15 | */ | 15 | */ |
16 | package org.onosproject.net.intent; | 16 | package org.onosproject.net.intent; |
17 | 17 | ||
18 | -import java.util.Collections; | 18 | +import com.google.common.base.MoreObjects; |
19 | -import java.util.List; | 19 | +import com.google.common.collect.ImmutableSet; |
20 | - | ||
21 | import org.onosproject.core.ApplicationId; | 20 | import org.onosproject.core.ApplicationId; |
22 | import org.onosproject.net.HostId; | 21 | import org.onosproject.net.HostId; |
23 | import org.onosproject.net.flow.TrafficSelector; | 22 | import org.onosproject.net.flow.TrafficSelector; |
24 | import org.onosproject.net.flow.TrafficTreatment; | 23 | import org.onosproject.net.flow.TrafficTreatment; |
25 | 24 | ||
26 | -import com.google.common.base.MoreObjects; | 25 | +import java.util.List; |
27 | 26 | ||
28 | import static com.google.common.base.Preconditions.checkNotNull; | 27 | import static com.google.common.base.Preconditions.checkNotNull; |
29 | 28 | ||
... | @@ -147,7 +146,7 @@ public final class HostToHostIntent extends ConnectivityIntent { | ... | @@ -147,7 +146,7 @@ public final class HostToHostIntent extends ConnectivityIntent { |
147 | TrafficTreatment treatment, | 146 | TrafficTreatment treatment, |
148 | List<Constraint> constraints, | 147 | List<Constraint> constraints, |
149 | int priority) { | 148 | int priority) { |
150 | - super(appId, key, Collections.emptyList(), selector, treatment, | 149 | + super(appId, key, ImmutableSet.of(one, two), selector, treatment, |
151 | constraints, priority); | 150 | constraints, priority); |
152 | 151 | ||
153 | // TODO: consider whether the case one and two are same is allowed | 152 | // TODO: consider whether the case one and two are same is allowed | ... | ... |
... | @@ -377,7 +377,9 @@ public class IntentManager | ... | @@ -377,7 +377,9 @@ public class IntentManager |
377 | throw new IllegalStateException("installable intents must be FlowRuleIntent"); | 377 | throw new IllegalStateException("installable intents must be FlowRuleIntent"); |
378 | } | 378 | } |
379 | 379 | ||
380 | - installables.forEach(x -> trackerService.addTrackedResources(data.key(), x.resources())); | 380 | + trackerService.addTrackedResources(data.key(), data.intent().resources()); |
381 | + installables.forEach(installable -> | ||
382 | + trackerService.addTrackedResources(data.key(), installable.resources())); | ||
381 | 383 | ||
382 | List<Collection<FlowRule>> stages = installables.stream() | 384 | List<Collection<FlowRule>> stages = installables.stream() |
383 | .map(x -> (FlowRuleIntent) x) | 385 | .map(x -> (FlowRuleIntent) x) |
... | @@ -415,7 +417,10 @@ public class IntentManager | ... | @@ -415,7 +417,10 @@ public class IntentManager |
415 | throw new IllegalStateException("installable intents must be FlowRuleIntent"); | 417 | throw new IllegalStateException("installable intents must be FlowRuleIntent"); |
416 | } | 418 | } |
417 | 419 | ||
418 | - installables.forEach(x -> trackerService.removeTrackedResources(data.intent().key(), x.resources())); | 420 | + trackerService.removeTrackedResources(data.key(), data.intent().resources()); |
421 | + installables.forEach(installable -> | ||
422 | + trackerService.removeTrackedResources(data.intent().key(), | ||
423 | + installable.resources())); | ||
419 | 424 | ||
420 | List<Collection<FlowRule>> stages = installables.stream() | 425 | List<Collection<FlowRule>> stages = installables.stream() |
421 | .map(x -> (FlowRuleIntent) x) | 426 | .map(x -> (FlowRuleIntent) x) | ... | ... |
... | @@ -26,9 +26,18 @@ import org.apache.felix.scr.annotations.ReferenceCardinality; | ... | @@ -26,9 +26,18 @@ import org.apache.felix.scr.annotations.ReferenceCardinality; |
26 | import org.apache.felix.scr.annotations.Service; | 26 | import org.apache.felix.scr.annotations.Service; |
27 | import org.onosproject.core.ApplicationId; | 27 | import org.onosproject.core.ApplicationId; |
28 | import org.onosproject.event.Event; | 28 | import org.onosproject.event.Event; |
29 | +import org.onosproject.net.DeviceId; | ||
30 | +import org.onosproject.net.ElementId; | ||
31 | +import org.onosproject.net.HostId; | ||
29 | import org.onosproject.net.Link; | 32 | import org.onosproject.net.Link; |
30 | import org.onosproject.net.LinkKey; | 33 | import org.onosproject.net.LinkKey; |
31 | import org.onosproject.net.NetworkResource; | 34 | import org.onosproject.net.NetworkResource; |
35 | +import org.onosproject.net.device.DeviceEvent; | ||
36 | +import org.onosproject.net.device.DeviceListener; | ||
37 | +import org.onosproject.net.device.DeviceService; | ||
38 | +import org.onosproject.net.host.HostEvent; | ||
39 | +import org.onosproject.net.host.HostListener; | ||
40 | +import org.onosproject.net.host.HostService; | ||
32 | import org.onosproject.net.intent.IntentService; | 41 | import org.onosproject.net.intent.IntentService; |
33 | import org.onosproject.net.intent.Key; | 42 | import org.onosproject.net.intent.Key; |
34 | import org.onosproject.net.link.LinkEvent; | 43 | import org.onosproject.net.link.LinkEvent; |
... | @@ -41,6 +50,7 @@ import org.onosproject.net.topology.TopologyService; | ... | @@ -41,6 +50,7 @@ import org.onosproject.net.topology.TopologyService; |
41 | import org.slf4j.Logger; | 50 | import org.slf4j.Logger; |
42 | 51 | ||
43 | import java.util.Collection; | 52 | import java.util.Collection; |
53 | +import java.util.Collections; | ||
44 | import java.util.HashSet; | 54 | import java.util.HashSet; |
45 | import java.util.Set; | 55 | import java.util.Set; |
46 | import java.util.concurrent.ExecutorService; | 56 | import java.util.concurrent.ExecutorService; |
... | @@ -69,27 +79,40 @@ public class ObjectiveTracker implements ObjectiveTrackerService { | ... | @@ -69,27 +79,40 @@ public class ObjectiveTracker implements ObjectiveTrackerService { |
69 | //TODO this could be slow as a point of synchronization | 79 | //TODO this could be slow as a point of synchronization |
70 | synchronizedSetMultimap(HashMultimap.<LinkKey, Key>create()); | 80 | synchronizedSetMultimap(HashMultimap.<LinkKey, Key>create()); |
71 | 81 | ||
82 | + private final SetMultimap<ElementId, Key> intentsByDevice = | ||
83 | + synchronizedSetMultimap(HashMultimap.<ElementId, Key>create()); | ||
84 | + | ||
72 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) | 85 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
73 | protected TopologyService topologyService; | 86 | protected TopologyService topologyService; |
74 | 87 | ||
75 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) | 88 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
76 | protected LinkResourceService resourceManager; | 89 | protected LinkResourceService resourceManager; |
77 | 90 | ||
91 | + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) | ||
92 | + protected DeviceService deviceService; | ||
93 | + | ||
94 | + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) | ||
95 | + protected HostService hostService; | ||
96 | + | ||
78 | @Reference(cardinality = ReferenceCardinality.OPTIONAL_UNARY) | 97 | @Reference(cardinality = ReferenceCardinality.OPTIONAL_UNARY) |
79 | protected IntentService intentService; | 98 | protected IntentService intentService; |
80 | 99 | ||
81 | private ExecutorService executorService = | 100 | private ExecutorService executorService = |
82 | - newSingleThreadExecutor(groupedThreads("onos/intent", "flowtracker")); | 101 | + newSingleThreadExecutor(groupedThreads("onos/intent", "objectivetracker")); |
83 | 102 | ||
84 | private TopologyListener listener = new InternalTopologyListener(); | 103 | private TopologyListener listener = new InternalTopologyListener(); |
85 | private LinkResourceListener linkResourceListener = | 104 | private LinkResourceListener linkResourceListener = |
86 | new InternalLinkResourceListener(); | 105 | new InternalLinkResourceListener(); |
106 | + private DeviceListener deviceListener = new InternalDeviceListener(); | ||
107 | + private HostListener hostListener = new InternalHostListener(); | ||
87 | private TopologyChangeDelegate delegate; | 108 | private TopologyChangeDelegate delegate; |
88 | 109 | ||
89 | @Activate | 110 | @Activate |
90 | public void activate() { | 111 | public void activate() { |
91 | topologyService.addListener(listener); | 112 | topologyService.addListener(listener); |
92 | resourceManager.addListener(linkResourceListener); | 113 | resourceManager.addListener(linkResourceListener); |
114 | + deviceService.addListener(deviceListener); | ||
115 | + hostService.addListener(hostListener); | ||
93 | log.info("Started"); | 116 | log.info("Started"); |
94 | } | 117 | } |
95 | 118 | ||
... | @@ -97,6 +120,8 @@ public class ObjectiveTracker implements ObjectiveTrackerService { | ... | @@ -97,6 +120,8 @@ public class ObjectiveTracker implements ObjectiveTrackerService { |
97 | public void deactivate() { | 120 | public void deactivate() { |
98 | topologyService.removeListener(listener); | 121 | topologyService.removeListener(listener); |
99 | resourceManager.removeListener(linkResourceListener); | 122 | resourceManager.removeListener(linkResourceListener); |
123 | + deviceService.removeListener(deviceListener); | ||
124 | + hostService.removeListener(hostListener); | ||
100 | log.info("Stopped"); | 125 | log.info("Stopped"); |
101 | } | 126 | } |
102 | 127 | ||
... | @@ -132,6 +157,8 @@ public class ObjectiveTracker implements ObjectiveTrackerService { | ... | @@ -132,6 +157,8 @@ public class ObjectiveTracker implements ObjectiveTrackerService { |
132 | for (NetworkResource resource : resources) { | 157 | for (NetworkResource resource : resources) { |
133 | if (resource instanceof Link) { | 158 | if (resource instanceof Link) { |
134 | intentsByLink.put(linkKey((Link) resource), intentKey); | 159 | intentsByLink.put(linkKey((Link) resource), intentKey); |
160 | + } else if (resource instanceof ElementId) { | ||
161 | + intentsByDevice.put((ElementId) resource, intentKey); | ||
135 | } | 162 | } |
136 | } | 163 | } |
137 | } | 164 | } |
... | @@ -142,6 +169,8 @@ public class ObjectiveTracker implements ObjectiveTrackerService { | ... | @@ -142,6 +169,8 @@ public class ObjectiveTracker implements ObjectiveTrackerService { |
142 | for (NetworkResource resource : resources) { | 169 | for (NetworkResource resource : resources) { |
143 | if (resource instanceof Link) { | 170 | if (resource instanceof Link) { |
144 | intentsByLink.remove(linkKey((Link) resource), intentKey); | 171 | intentsByLink.remove(linkKey((Link) resource), intentKey); |
172 | + } else if (resource instanceof ElementId) { | ||
173 | + intentsByDevice.remove((ElementId) resource, intentKey); | ||
145 | } | 174 | } |
146 | } | 175 | } |
147 | } | 176 | } |
... | @@ -171,7 +200,7 @@ public class ObjectiveTracker implements ObjectiveTrackerService { | ... | @@ -171,7 +200,7 @@ public class ObjectiveTracker implements ObjectiveTrackerService { |
171 | } | 200 | } |
172 | 201 | ||
173 | if (event.reasons() == null || event.reasons().isEmpty()) { | 202 | if (event.reasons() == null || event.reasons().isEmpty()) { |
174 | - delegate.triggerCompile(new HashSet<Key>(), true); | 203 | + delegate.triggerCompile(Collections.emptySet(), true); |
175 | 204 | ||
176 | } else { | 205 | } else { |
177 | Set<Key> toBeRecompiled = new HashSet<>(); | 206 | Set<Key> toBeRecompiled = new HashSet<>(); |
... | @@ -231,7 +260,7 @@ public class ObjectiveTracker implements ObjectiveTrackerService { | ... | @@ -231,7 +260,7 @@ public class ObjectiveTracker implements ObjectiveTrackerService { |
231 | return; | 260 | return; |
232 | } | 261 | } |
233 | 262 | ||
234 | - delegate.triggerCompile(new HashSet<>(), true); | 263 | + delegate.triggerCompile(Collections.emptySet(), true); |
235 | } | 264 | } |
236 | } | 265 | } |
237 | 266 | ||
... | @@ -257,4 +286,60 @@ public class ObjectiveTracker implements ObjectiveTrackerService { | ... | @@ -257,4 +286,60 @@ public class ObjectiveTracker implements ObjectiveTrackerService { |
257 | } | 286 | } |
258 | }); | 287 | }); |
259 | } | 288 | } |
289 | + | ||
290 | + /* | ||
291 | + * Re-dispatcher of device and host events. | ||
292 | + */ | ||
293 | + private class DeviceAvailabilityHandler implements Runnable { | ||
294 | + | ||
295 | + private final ElementId id; | ||
296 | + private final boolean available; | ||
297 | + | ||
298 | + DeviceAvailabilityHandler(ElementId id, boolean available) { | ||
299 | + this.id = checkNotNull(id); | ||
300 | + this.available = available; | ||
301 | + } | ||
302 | + | ||
303 | + @Override | ||
304 | + public void run() { | ||
305 | + // If there is no delegate, why bother? Just bail. | ||
306 | + if (delegate == null) { | ||
307 | + return; | ||
308 | + } | ||
309 | + | ||
310 | + // TODO should we recompile on available==true? | ||
311 | + delegate.triggerCompile(intentsByDevice.get(id), available); | ||
312 | + } | ||
313 | + } | ||
314 | + | ||
315 | + | ||
316 | + private class InternalDeviceListener implements DeviceListener { | ||
317 | + @Override | ||
318 | + public void event(DeviceEvent event) { | ||
319 | + DeviceEvent.Type type = event.type(); | ||
320 | + if (type == DeviceEvent.Type.PORT_ADDED || | ||
321 | + type == DeviceEvent.Type.PORT_UPDATED || | ||
322 | + type == DeviceEvent.Type.PORT_REMOVED) { | ||
323 | + // skip port events for now | ||
324 | + return; | ||
325 | + } | ||
326 | + DeviceId id = event.subject().id(); | ||
327 | + // TODO we need to check whether AVAILABILITY_CHANGED means up or down | ||
328 | + boolean available = (type == DeviceEvent.Type.DEVICE_AVAILABILITY_CHANGED || | ||
329 | + type == DeviceEvent.Type.DEVICE_ADDED || | ||
330 | + type == DeviceEvent.Type.DEVICE_UPDATED); | ||
331 | + executorService.execute(new DeviceAvailabilityHandler(id, available)); | ||
332 | + | ||
333 | + } | ||
334 | + } | ||
335 | + | ||
336 | + private class InternalHostListener implements HostListener { | ||
337 | + @Override | ||
338 | + public void event(HostEvent event) { | ||
339 | + HostId id = event.subject().id(); | ||
340 | + HostEvent.Type type = event.type(); | ||
341 | + boolean available = (type == HostEvent.Type.HOST_ADDED); | ||
342 | + executorService.execute(new DeviceAvailabilityHandler(id, available)); | ||
343 | + } | ||
344 | + } | ||
260 | } | 345 | } | ... | ... |
... | @@ -29,8 +29,11 @@ import org.onlab.junit.TestUtils; | ... | @@ -29,8 +29,11 @@ import org.onlab.junit.TestUtils; |
29 | import org.onlab.junit.TestUtils.TestUtilsException; | 29 | import org.onlab.junit.TestUtils.TestUtilsException; |
30 | import org.onosproject.core.IdGenerator; | 30 | import org.onosproject.core.IdGenerator; |
31 | import org.onosproject.event.Event; | 31 | import org.onosproject.event.Event; |
32 | +import org.onosproject.net.Device; | ||
32 | import org.onosproject.net.Link; | 33 | import org.onosproject.net.Link; |
33 | import org.onosproject.net.NetworkResource; | 34 | import org.onosproject.net.NetworkResource; |
35 | +import org.onosproject.net.device.DeviceEvent; | ||
36 | +import org.onosproject.net.device.DeviceListener; | ||
34 | import org.onosproject.net.intent.Intent; | 37 | import org.onosproject.net.intent.Intent; |
35 | import org.onosproject.net.intent.Key; | 38 | import org.onosproject.net.intent.Key; |
36 | import org.onosproject.net.intent.MockIdGenerator; | 39 | import org.onosproject.net.intent.MockIdGenerator; |
... | @@ -50,6 +53,7 @@ import static org.hamcrest.Matchers.equalTo; | ... | @@ -50,6 +53,7 @@ import static org.hamcrest.Matchers.equalTo; |
50 | import static org.hamcrest.Matchers.hasSize; | 53 | import static org.hamcrest.Matchers.hasSize; |
51 | import static org.hamcrest.Matchers.is; | 54 | import static org.hamcrest.Matchers.is; |
52 | import static org.onosproject.net.NetTestTools.APP_ID; | 55 | import static org.onosproject.net.NetTestTools.APP_ID; |
56 | +import static org.onosproject.net.NetTestTools.device; | ||
53 | import static org.onosproject.net.NetTestTools.link; | 57 | import static org.onosproject.net.NetTestTools.link; |
54 | 58 | ||
55 | /** | 59 | /** |
... | @@ -62,6 +66,7 @@ public class ObjectiveTrackerTest { | ... | @@ -62,6 +66,7 @@ public class ObjectiveTrackerTest { |
62 | private TestTopologyChangeDelegate delegate; | 66 | private TestTopologyChangeDelegate delegate; |
63 | private List<Event> reasons; | 67 | private List<Event> reasons; |
64 | private TopologyListener listener; | 68 | private TopologyListener listener; |
69 | + private DeviceListener deviceListener; | ||
65 | private LinkResourceListener linkResourceListener; | 70 | private LinkResourceListener linkResourceListener; |
66 | private IdGenerator mockGenerator; | 71 | private IdGenerator mockGenerator; |
67 | 72 | ||
... | @@ -78,6 +83,7 @@ public class ObjectiveTrackerTest { | ... | @@ -78,6 +83,7 @@ public class ObjectiveTrackerTest { |
78 | tracker.setDelegate(delegate); | 83 | tracker.setDelegate(delegate); |
79 | reasons = new LinkedList<>(); | 84 | reasons = new LinkedList<>(); |
80 | listener = TestUtils.getField(tracker, "listener"); | 85 | listener = TestUtils.getField(tracker, "listener"); |
86 | + deviceListener = TestUtils.getField(tracker, "deviceListener"); | ||
81 | linkResourceListener = TestUtils.getField(tracker, "linkResourceListener"); | 87 | linkResourceListener = TestUtils.getField(tracker, "linkResourceListener"); |
82 | mockGenerator = new MockIdGenerator(); | 88 | mockGenerator = new MockIdGenerator(); |
83 | Intent.bindIdGenerator(mockGenerator); | 89 | Intent.bindIdGenerator(mockGenerator); |
... | @@ -235,4 +241,86 @@ public class ObjectiveTrackerTest { | ... | @@ -235,4 +241,86 @@ public class ObjectiveTrackerTest { |
235 | assertThat(delegate.compileAllFailedFromEvent, is(true)); | 241 | assertThat(delegate.compileAllFailedFromEvent, is(true)); |
236 | } | 242 | } |
237 | 243 | ||
244 | + /** | ||
245 | + * Tests an event for a host becoming available that matches an intent. | ||
246 | + * | ||
247 | + * @throws InterruptedException if the latch wait fails. | ||
248 | + */ | ||
249 | + | ||
250 | + @Test | ||
251 | + public void testEventHostAvailableMatch() throws Exception { | ||
252 | + final Device host = device("host1"); | ||
253 | + | ||
254 | + final DeviceEvent deviceEvent = | ||
255 | + new DeviceEvent(DeviceEvent.Type.DEVICE_ADDED, host); | ||
256 | + reasons.add(deviceEvent); | ||
257 | + | ||
258 | + final Key key = Key.of(0x333L, APP_ID); | ||
259 | + Collection<NetworkResource> resources = ImmutableSet.of(host.id()); | ||
260 | + tracker.addTrackedResources(key, resources); | ||
261 | + | ||
262 | + deviceListener.event(deviceEvent); | ||
263 | + assertThat( | ||
264 | + delegate.latch.await(WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS), | ||
265 | + is(true)); | ||
266 | + | ||
267 | + assertThat(delegate.intentIdsFromEvent, hasSize(1)); | ||
268 | + assertThat(delegate.compileAllFailedFromEvent, is(true)); | ||
269 | + assertThat(delegate.intentIdsFromEvent.get(0).toString(), | ||
270 | + equalTo("0x333")); | ||
271 | + } | ||
272 | + | ||
273 | + /** | ||
274 | + * Tests an event for a host becoming unavailable that matches an intent. | ||
275 | + * | ||
276 | + * @throws InterruptedException if the latch wait fails. | ||
277 | + */ | ||
278 | + | ||
279 | + @Test | ||
280 | + public void testEventHostUnavailableMatch() throws Exception { | ||
281 | + final Device host = device("host1"); | ||
282 | + | ||
283 | + final DeviceEvent deviceEvent = | ||
284 | + new DeviceEvent(DeviceEvent.Type.DEVICE_REMOVED, host); | ||
285 | + reasons.add(deviceEvent); | ||
286 | + | ||
287 | + final Key key = Key.of(0x333L, APP_ID); | ||
288 | + Collection<NetworkResource> resources = ImmutableSet.of(host.id()); | ||
289 | + tracker.addTrackedResources(key, resources); | ||
290 | + | ||
291 | + deviceListener.event(deviceEvent); | ||
292 | + assertThat( | ||
293 | + delegate.latch.await(WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS), | ||
294 | + is(true)); | ||
295 | + | ||
296 | + assertThat(delegate.intentIdsFromEvent, hasSize(1)); | ||
297 | + assertThat(delegate.compileAllFailedFromEvent, is(false)); | ||
298 | + assertThat(delegate.intentIdsFromEvent.get(0).toString(), | ||
299 | + equalTo("0x333")); | ||
300 | + } | ||
301 | + | ||
302 | + /** | ||
303 | + * Tests an event for a host becoming available that matches an intent. | ||
304 | + * | ||
305 | + * @throws InterruptedException if the latch wait fails. | ||
306 | + */ | ||
307 | + | ||
308 | + @Test | ||
309 | + public void testEventHostAvailableNoMatch() throws Exception { | ||
310 | + final Device host = device("host1"); | ||
311 | + | ||
312 | + final DeviceEvent deviceEvent = | ||
313 | + new DeviceEvent(DeviceEvent.Type.DEVICE_ADDED, host); | ||
314 | + reasons.add(deviceEvent); | ||
315 | + | ||
316 | + deviceListener.event(deviceEvent); | ||
317 | + assertThat( | ||
318 | + delegate.latch.await(WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS), | ||
319 | + is(true)); | ||
320 | + | ||
321 | + assertThat(delegate.intentIdsFromEvent, hasSize(0)); | ||
322 | + assertThat(delegate.compileAllFailedFromEvent, is(true)); | ||
323 | + } | ||
324 | + | ||
325 | + | ||
238 | } | 326 | } | ... | ... |
-
Please register or login to post a comment