홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Charles Chan
Committed by Gerrit Code Review
2199c30c 1 parent a78cdb21

CORD-348 Fabric multicast support - error handling 

Automatically failover to backup spine if
- ingress - transit link down
- transit - egress link down
- transit device down

Can recover from fatal error with human involved
- ingress switch down
- egress switch down
- all links to spine down

Scan through McastRouteStore when
- SR activate
- link up

Also include following features
- Use flow objective context in McastHandler
- Update Mcast VLAN config sample

Change-Id: I75007d9efd7646e7c4e57fa6d3fc6943543153cf
Expand all
Inline Side-by-side
Showing 5 changed files with 45 additions and 66 deletions
...@@ -762,6 +762,8 @@ public class SegmentRoutingManager implements SegmentRoutingService { ...@@ -762,6 +762,8 @@ public class SegmentRoutingManager implements SegmentRoutingService {
762 defaultRoutingHandler.populateRoutingRulesForLinkStatusChange(null); 762 defaultRoutingHandler.populateRoutingRulesForLinkStatusChange(null);
763 //log.trace("processLinkAdded: re-starting route population process"); 763 //log.trace("processLinkAdded: re-starting route population process");
764 //defaultRoutingHandler.startPopulationProcess(); 764 //defaultRoutingHandler.startPopulationProcess();
765 +
766 + mcastHandler.init();
765 } 767 }
766 768
767 private void processLinkRemoved(Link link) { 769 private void processLinkRemoved(Link link) {
...@@ -775,6 +777,8 @@ public class SegmentRoutingManager implements SegmentRoutingService { ...@@ -775,6 +777,8 @@ public class SegmentRoutingManager implements SegmentRoutingService {
775 defaultRoutingHandler.populateRoutingRulesForLinkStatusChange(link); 777 defaultRoutingHandler.populateRoutingRulesForLinkStatusChange(link);
776 //log.trace("processLinkRemoved: re-starting route population process"); 778 //log.trace("processLinkRemoved: re-starting route population process");
777 //defaultRoutingHandler.startPopulationProcess(); 779 //defaultRoutingHandler.startPopulationProcess();
780 +
781 + mcastHandler.processLinkDown(link);
778 } 782 }
779 783
780 private void processDeviceAdded(Device device) { 784 private void processDeviceAdded(Device device) {
...@@ -784,17 +788,22 @@ public class SegmentRoutingManager implements SegmentRoutingService { ...@@ -784,17 +788,22 @@ public class SegmentRoutingManager implements SegmentRoutingService {
784 + "processed after config completes.", device.id()); 788 + "processed after config completes.", device.id());
785 return; 789 return;
786 } 790 }
791 + processDeviceAddedInternal(device.id());
792 + }
793 +
794 + private void processDeviceAddedInternal(DeviceId deviceId) {
787 // Irrespective of whether the local is a MASTER or not for this device, 795 // Irrespective of whether the local is a MASTER or not for this device,
788 // we need to create a SR-group-handler instance. This is because in a 796 // we need to create a SR-group-handler instance. This is because in a
789 // multi-instance setup, any instance can initiate forwarding/next-objectives 797 // multi-instance setup, any instance can initiate forwarding/next-objectives
790 // for any switch (even if this instance is a SLAVE or not even connected 798 // for any switch (even if this instance is a SLAVE or not even connected
791 // to the switch). To handle this, a default-group-handler instance is necessary 799 // to the switch). To handle this, a default-group-handler instance is necessary
792 // per switch. 800 // per switch.
793 - if (groupHandlerMap.get(device.id()) == null) { 801 + log.debug("Current groupHandlerMap devs: {}", groupHandlerMap.keySet());
802 + if (groupHandlerMap.get(deviceId) == null) {
794 DefaultGroupHandler groupHandler; 803 DefaultGroupHandler groupHandler;
795 try { 804 try {
796 groupHandler = DefaultGroupHandler. 805 groupHandler = DefaultGroupHandler.
797 - createGroupHandler(device.id(), 806 + createGroupHandler(deviceId,
798 appId, 807 appId,
799 deviceConfiguration, 808 deviceConfiguration,
800 linkService, 809 linkService,
...@@ -804,23 +813,25 @@ public class SegmentRoutingManager implements SegmentRoutingService { ...@@ -804,23 +813,25 @@ public class SegmentRoutingManager implements SegmentRoutingService {
804 log.warn(e.getMessage() + " Aborting processDeviceAdded."); 813 log.warn(e.getMessage() + " Aborting processDeviceAdded.");
805 return; 814 return;
806 } 815 }
807 - groupHandlerMap.put(device.id(), groupHandler); 816 + log.debug("updating groupHandlerMap with new config for device: {}",
817 + deviceId);
818 + groupHandlerMap.put(deviceId, groupHandler);
808 // Also, in some cases, drivers may need extra 819 // Also, in some cases, drivers may need extra
809 // information to process rules (eg. Router IP/MAC); and so, we send 820 // information to process rules (eg. Router IP/MAC); and so, we send
810 // port addressing rules to the driver as well irrespective of whether 821 // port addressing rules to the driver as well irrespective of whether
811 // this instance is the master or not. 822 // this instance is the master or not.
812 - defaultRoutingHandler.populatePortAddressingRules(device.id()); 823 + defaultRoutingHandler.populatePortAddressingRules(deviceId);
813 } 824 }
814 - if (mastershipService.isLocalMaster(device.id())) { 825 + if (mastershipService.isLocalMaster(deviceId)) {
815 - hostHandler.readInitialHosts(device.id()); 826 + hostHandler.readInitialHosts(deviceId);
816 - DefaultGroupHandler groupHandler = groupHandlerMap.get(device.id()); 827 + DefaultGroupHandler groupHandler = groupHandlerMap.get(deviceId);
817 groupHandler.createGroupsFromSubnetConfig(); 828 groupHandler.createGroupsFromSubnetConfig();
818 - routingRulePopulator.populateSubnetBroadcastRule(device.id()); 829 + routingRulePopulator.populateSubnetBroadcastRule(deviceId);
819 - groupHandler.createGroupsForXConnect(device.id()); 830 + groupHandler.createGroupsForXConnect(deviceId);
820 - routingRulePopulator.populateXConnectBroadcastRule(device.id()); 831 + routingRulePopulator.populateXConnectBroadcastRule(deviceId);
821 } 832 }
822 833
823 - netcfgHandler.initVRouters(device.id()); 834 + netcfgHandler.initVRouters(deviceId);
824 } 835 }
825 836
826 private void processDeviceRemoved(Device device) { 837 private void processDeviceRemoved(Device device) {
...@@ -829,34 +840,29 @@ public class SegmentRoutingManager implements SegmentRoutingService { ...@@ -829,34 +840,29 @@ public class SegmentRoutingManager implements SegmentRoutingService {
829 .forEach(entry -> { 840 .forEach(entry -> {
830 nsNextObjStore.remove(entry.getKey()); 841 nsNextObjStore.remove(entry.getKey());
831 }); 842 });
832 -
833 subnetNextObjStore.entrySet().stream() 843 subnetNextObjStore.entrySet().stream()
834 .filter(entry -> entry.getKey().deviceId().equals(device.id())) 844 .filter(entry -> entry.getKey().deviceId().equals(device.id()))
835 .forEach(entry -> { 845 .forEach(entry -> {
836 subnetNextObjStore.remove(entry.getKey()); 846 subnetNextObjStore.remove(entry.getKey());
837 }); 847 });
838 -
839 portNextObjStore.entrySet().stream() 848 portNextObjStore.entrySet().stream()
840 .filter(entry -> entry.getKey().deviceId().equals(device.id())) 849 .filter(entry -> entry.getKey().deviceId().equals(device.id()))
841 .forEach(entry -> { 850 .forEach(entry -> {
842 portNextObjStore.remove(entry.getKey()); 851 portNextObjStore.remove(entry.getKey());
843 }); 852 });
844 -
845 xConnectNextObjStore.entrySet().stream() 853 xConnectNextObjStore.entrySet().stream()
846 .filter(entry -> entry.getKey().deviceId().equals(device.id())) 854 .filter(entry -> entry.getKey().deviceId().equals(device.id()))
847 .forEach(entry -> { 855 .forEach(entry -> {
848 xConnectNextObjStore.remove(entry.getKey()); 856 xConnectNextObjStore.remove(entry.getKey());
849 }); 857 });
850 -
851 subnetVidStore.entrySet().stream() 858 subnetVidStore.entrySet().stream()
852 .filter(entry -> entry.getKey().deviceId().equals(device.id())) 859 .filter(entry -> entry.getKey().deviceId().equals(device.id()))
853 .forEach(entry -> { 860 .forEach(entry -> {
854 subnetVidStore.remove(entry.getKey()); 861 subnetVidStore.remove(entry.getKey());
855 }); 862 });
856 -
857 groupHandlerMap.remove(device.id()); 863 groupHandlerMap.remove(device.id());
858 -
859 defaultRoutingHandler.purgeEcmpGraph(device.id()); 864 defaultRoutingHandler.purgeEcmpGraph(device.id());
865 + mcastHandler.removeDevice(device.id());
860 } 866 }
861 867
862 private void processPortRemoved(Device device, Port port) { 868 private void processPortRemoved(Device device, Port port) {
...@@ -900,48 +906,11 @@ public class SegmentRoutingManager implements SegmentRoutingService { ...@@ -900,48 +906,11 @@ public class SegmentRoutingManager implements SegmentRoutingService {
900 tunnelHandler, policyStore); 906 tunnelHandler, policyStore);
901 907
902 for (Device device : deviceService.getDevices()) { 908 for (Device device : deviceService.getDevices()) {
903 - // Irrespective of whether the local is a MASTER or not for this device, 909 + processDeviceAddedInternal(device.id());
904 - // we need to create a SR-group-handler instance. This is because in a
905 - // multi-instance setup, any instance can initiate forwarding/next-objectives
906 - // for any switch (even if this instance is a SLAVE or not even connected
907 - // to the switch). To handle this, a default-group-handler instance is necessary
908 - // per switch.
909 - log.debug("Current groupHandlerMap devs: {}", groupHandlerMap.keySet());
910 - if (groupHandlerMap.get(device.id()) == null) {
911 - DefaultGroupHandler groupHandler;
912 - try {
913 - groupHandler = DefaultGroupHandler.
914 - createGroupHandler(device.id(),
915 - appId,
916 - deviceConfiguration,
917 - linkService,
918 - flowObjectiveService,
919 - segmentRoutingManager);
920 - } catch (DeviceConfigNotFoundException e) {
921 - log.warn(e.getMessage() + " Aborting configureNetwork.");
922 - return;
923 - }
924 - log.debug("updating groupHandlerMap with new config for "
925 - + "device: {}", device.id());
926 - groupHandlerMap.put(device.id(), groupHandler);
927 -
928 - // Also, in some cases, drivers may need extra
929 - // information to process rules (eg. Router IP/MAC); and so, we send
930 - // port addressing rules to the driver as well, irrespective of whether
931 - // this instance is the master or not.
932 - defaultRoutingHandler.populatePortAddressingRules(device.id());
933 - }
934 - if (mastershipService.isLocalMaster(device.id())) {
935 - hostHandler.readInitialHosts(device.id());
936 - DefaultGroupHandler groupHandler = groupHandlerMap.get(device.id());
937 - groupHandler.createGroupsFromSubnetConfig();
938 - routingRulePopulator.populateSubnetBroadcastRule(device.id());
939 - groupHandler.createGroupsForXConnect(device.id());
940 - routingRulePopulator.populateXConnectBroadcastRule(device.id());
941 - }
942 } 910 }
943 911
944 defaultRoutingHandler.startPopulationProcess(); 912 defaultRoutingHandler.startPopulationProcess();
913 + mcastHandler.init();
945 } 914 }
946 915
947 @Override 916 @Override
......
...@@ -26,7 +26,7 @@ import java.util.Objects; ...@@ -26,7 +26,7 @@ import java.util.Objects;
26 /** 26 /**
27 * Key of multicast next objective store. 27 * Key of multicast next objective store.
28 */ 28 */
29 -public class McastNextObjectiveStoreKey { 29 +public class McastStoreKey {
30 private final IpAddress mcastIp; 30 private final IpAddress mcastIp;
31 private final DeviceId deviceId; 31 private final DeviceId deviceId;
32 32
...@@ -36,7 +36,7 @@ public class McastNextObjectiveStoreKey { ...@@ -36,7 +36,7 @@ public class McastNextObjectiveStoreKey {
36 * @param mcastIp multicast group IP address 36 * @param mcastIp multicast group IP address
37 * @param deviceId device ID 37 * @param deviceId device ID
38 */ 38 */
39 - public McastNextObjectiveStoreKey(IpAddress mcastIp, DeviceId deviceId) { 39 + public McastStoreKey(IpAddress mcastIp, DeviceId deviceId) {
40 checkNotNull(mcastIp, "mcastIp cannot be null"); 40 checkNotNull(mcastIp, "mcastIp cannot be null");
41 checkNotNull(deviceId, "deviceId cannot be null"); 41 checkNotNull(deviceId, "deviceId cannot be null");
42 checkArgument(mcastIp.isMulticast(), "mcastIp must be a multicast address"); 42 checkArgument(mcastIp.isMulticast(), "mcastIp must be a multicast address");
...@@ -67,11 +67,11 @@ public class McastNextObjectiveStoreKey { ...@@ -67,11 +67,11 @@ public class McastNextObjectiveStoreKey {
67 if (this == o) { 67 if (this == o) {
68 return true; 68 return true;
69 } 69 }
70 - if (!(o instanceof McastNextObjectiveStoreKey)) { 70 + if (!(o instanceof McastStoreKey)) {
71 return false; 71 return false;
72 } 72 }
73 - McastNextObjectiveStoreKey that = 73 + McastStoreKey that =
74 - (McastNextObjectiveStoreKey) o; 74 + (McastStoreKey) o;
75 return (Objects.equals(this.mcastIp, that.mcastIp) && 75 return (Objects.equals(this.mcastIp, that.mcastIp) &&
76 Objects.equals(this.deviceId, that.deviceId)); 76 Objects.equals(this.deviceId, that.deviceId));
77 } 77 }
......
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
5 <meta http-equiv="Content-Style-Type" content="text/css"> 5 <meta http-equiv="Content-Style-Type" content="text/css">
6 <title></title> 6 <title></title>
7 <meta name="Generator" content="Cocoa HTML Writer"> 7 <meta name="Generator" content="Cocoa HTML Writer">
8 - <meta name="CocoaVersion" content="1404.34"> 8 + <meta name="CocoaVersion" content="1404.46">
9 <style type="text/css"> 9 <style type="text/css">
10 p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; line-height: 14.0px; font: 12.0px Menlo; color: #0433ff; -webkit-text-stroke: #0433ff} 10 p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; line-height: 14.0px; font: 12.0px Menlo; color: #0433ff; -webkit-text-stroke: #0433ff}
11 p.p2 {margin: 0.0px 0.0px 0.0px 0.0px; line-height: 14.0px; font: 12.0px Menlo; color: #ff9300; -webkit-text-stroke: #ff9300} 11 p.p2 {margin: 0.0px 0.0px 0.0px 0.0px; line-height: 14.0px; font: 12.0px Menlo; color: #ff9300; -webkit-text-stroke: #ff9300}
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
14 p.p5 {margin: 0.0px 0.0px 0.0px 0.0px; line-height: 14.0px; font: 12.0px Menlo; color: #00c7fc; -webkit-text-stroke: #00c7fc} 14 p.p5 {margin: 0.0px 0.0px 0.0px 0.0px; line-height: 14.0px; font: 12.0px Menlo; color: #00c7fc; -webkit-text-stroke: #00c7fc}
15 p.p6 {margin: 0.0px 0.0px 0.0px 0.0px; line-height: 14.0px; font: 12.0px Menlo; color: #ff2600; -webkit-text-stroke: #ff2600} 15 p.p6 {margin: 0.0px 0.0px 0.0px 0.0px; line-height: 14.0px; font: 12.0px Menlo; color: #ff2600; -webkit-text-stroke: #ff2600}
16 p.p7 {margin: 0.0px 0.0px 0.0px 0.0px; line-height: 14.0px; font: 12.0px Menlo; color: #000000; -webkit-text-stroke: #000000} 16 p.p7 {margin: 0.0px 0.0px 0.0px 0.0px; line-height: 14.0px; font: 12.0px Menlo; color: #000000; -webkit-text-stroke: #000000}
17 + p.p8 {margin: 0.0px 0.0px 0.0px 0.0px; line-height: 14.0px; font: 12.0px Menlo; color: #00c7fc; -webkit-text-stroke: #000000}
17 span.s1 {font-kerning: none} 18 span.s1 {font-kerning: none}
18 span.s2 {font-kerning: none; color: #0433ff; -webkit-text-stroke: 0px #0433ff} 19 span.s2 {font-kerning: none; color: #0433ff; -webkit-text-stroke: 0px #0433ff}
19 span.s3 {font-kerning: none; color: #000000; -webkit-text-stroke: 0px #000000} 20 span.s3 {font-kerning: none; color: #000000; -webkit-text-stroke: 0px #000000}
...@@ -22,7 +23,8 @@ ...@@ -22,7 +23,8 @@
22 span.s6 {font-kerning: none; color: #77bb41; -webkit-text-stroke: 0px #77bb41} 23 span.s6 {font-kerning: none; color: #77bb41; -webkit-text-stroke: 0px #77bb41}
23 span.s7 {font-kerning: none; color: #00c7fc; -webkit-text-stroke: 0px #00c7fc} 24 span.s7 {font-kerning: none; color: #00c7fc; -webkit-text-stroke: 0px #00c7fc}
24 span.s8 {font-kerning: none; color: #ff2600; -webkit-text-stroke: 0px #ff2600} 25 span.s8 {font-kerning: none; color: #ff2600; -webkit-text-stroke: 0px #ff2600}
25 - span.s9 {font-kerning: none; color: #669c35; -webkit-text-stroke: 0px #669c35} 26 + span.s9 {font-kerning: none; color: #000000}
27 + span.s10 {font-kerning: none; color: #669c35; -webkit-text-stroke: 0px #669c35}
26 span.Apple-tab-span {white-space:pre} 28 span.Apple-tab-span {white-space:pre}
27 </style> 29 </style>
28 </head> 30 </head>
...@@ -31,7 +33,7 @@ ...@@ -31,7 +33,7 @@
31 <p class="p2"><span class="s1">Orange: vSG LAN connectivity (cross-connect)</span></p> 33 <p class="p2"><span class="s1">Orange: vSG LAN connectivity (cross-connect)</span></p>
32 <p class="p3"><span class="s1">Magenta: vSG WAN connectivity (default route)</span></p> 34 <p class="p3"><span class="s1">Magenta: vSG WAN connectivity (default route)</span></p>
33 <p class="p4"><span class="s1">Green: vRouter integration</span></p> 35 <p class="p4"><span class="s1">Green: vRouter integration</span></p>
34 -<p class="p5"><span class="s1">Light Blue: PIM integration</span></p> 36 +<p class="p5"><span class="s1">Light Blue: Multicast</span></p>
35 <p class="p6"><span class="s1">Red: Link validation</span></p> 37 <p class="p6"><span class="s1">Red: Link validation</span></p>
36 <p class="p7"><span class="s1"><br> 38 <p class="p7"><span class="s1"><br>
37 </span></p> 39 </span></p>
...@@ -221,7 +223,11 @@ ...@@ -221,7 +223,11 @@
221 <p class="p7"><span class="s1"><span class="Apple-converted-space">        </span>"org.onosproject.core" : {</span></p> 223 <p class="p7"><span class="s1"><span class="Apple-converted-space">        </span>"org.onosproject.core" : {</span></p>
222 <p class="p7"><span class="s1"><span class="Apple-converted-space">            </span>"core" : {</span></p> 224 <p class="p7"><span class="s1"><span class="Apple-converted-space">            </span>"core" : {</span></p>
223 <p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"linkDiscoveryMode" : "STRICT" </span><span class="s8">// enable strict link validation</span></p> 225 <p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"linkDiscoveryMode" : "STRICT" </span><span class="s8">// enable strict link validation</span></p>
224 -<p class="p7"><span class="s1"><span class="Apple-converted-space">            </span>} <span class="Apple-converted-space">   </span></span></p> 226 +<p class="p7"><span class="s1"><span class="Apple-converted-space">            </span>},</span></p>
227 +<p class="p8"><span class="s9"><span class="Apple-tab-span"> </span><span class="Apple-tab-span"> </span><span class="Apple-converted-space">  </span>"multicast": { </span><span class="s1">// The VLAN we expect to see on the multicast ingress and egress</span></p>
228 +<p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"ingressVlan": "None",</span></p>
229 +<p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"egressVlan": "None"</span></p>
230 +<p class="p7"><span class="s1"><span class="Apple-converted-space">            </span>}<span class="Apple-converted-space"> </span></span></p>
225 <p class="p7"><span class="s1"><span class="Apple-converted-space">        </span>},</span></p> 231 <p class="p7"><span class="s1"><span class="Apple-converted-space">        </span>},</span></p>
226 <p class="p7"><span class="s1"><span class="Apple-converted-space">        </span>"org.onosproject.segmentrouting" : {</span></p> 232 <p class="p7"><span class="s1"><span class="Apple-converted-space">        </span>"org.onosproject.segmentrouting" : {</span></p>
227 <p class="p7"><span class="s1"><span class="Apple-converted-space">            </span>"segmentrouting" : {</span></p> 233 <p class="p7"><span class="s1"><span class="Apple-converted-space">            </span>"segmentrouting" : {</span></p>
...@@ -243,7 +249,7 @@ ...@@ -243,7 +249,7 @@
243 <p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"controlPlaneConnectPoint" : "of:0000000000000002/31", </span><span class="s6">// location of Quagga</span></p> 249 <p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"controlPlaneConnectPoint" : "of:0000000000000002/31", </span><span class="s6">// location of Quagga</span></p>
244 <p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"ospfEnabled" : "true", </span><span class="s6">// enable OSPF</span></p> 250 <p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"ospfEnabled" : "true", </span><span class="s6">// enable OSPF</span></p>
245 <p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"pimEnabled" : "true", </span><span class="s7">// enable PIM</span></p> 251 <p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"pimEnabled" : "true", </span><span class="s7">// enable PIM</span></p>
246 -<p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"interfaces" : [ "external-quagga" ] </span><span class="s9">// </span><span class="s6">VR only handles peers on these ports</span></p> 252 +<p class="p7"><span class="s1"><span class="Apple-converted-space">                </span>"interfaces" : [ "external-quagga" ] </span><span class="s10">// </span><span class="s6">VR only handles peers on these ports</span></p>
247 <p class="p7"><span class="s1"><span class="Apple-converted-space">            </span>}</span></p> 253 <p class="p7"><span class="s1"><span class="Apple-converted-space">            </span>}</span></p>
248 <p class="p7"><span class="s1"><span class="Apple-converted-space">        </span>}</span></p> 254 <p class="p7"><span class="s1"><span class="Apple-converted-space">        </span>}</span></p>
249 <p class="p7"><span class="s1"><span class="Apple-converted-space">    </span>}</span></p> 255 <p class="p7"><span class="s1"><span class="Apple-converted-space">    </span>}</span></p>
......
...@@ -228,6 +228,10 @@ ...@@ -228,6 +228,10 @@
228 "org.onosproject.core" : { 228 "org.onosproject.core" : {
229 "core" : { 229 "core" : {
230 "linkDiscoveryMode" : "STRICT" 230 "linkDiscoveryMode" : "STRICT"
231 + },
232 + "multicast": {
233 + "ingressVlan": "None",
234 + "egressVlan": "None"
231 } 235 }
232 }, 236 },
233 "org.onosproject.segmentrouting" : { 237 "org.onosproject.segmentrouting" : {
......