Committed by
Gerrit Code Review
[Emu] [ONOS-2591,ONOS-2594] Implementation of BGP channel handler to manage each…
… BGP peer connection. Change-Id: I14e90c9437f676698f89da79e736a81035689492
Showing
9 changed files
with
1376 additions
and
32 deletions
... | @@ -25,6 +25,21 @@ import org.onosproject.bgpio.protocol.BGPMessage; | ... | @@ -25,6 +25,21 @@ import org.onosproject.bgpio.protocol.BGPMessage; |
25 | public interface BGPController { | 25 | public interface BGPController { |
26 | 26 | ||
27 | /** | 27 | /** |
28 | + * Returns list of bgp peers connected to this BGP controller. | ||
29 | + * | ||
30 | + * @return Iterable of BGPPeer elements | ||
31 | + */ | ||
32 | + Iterable<BGPPeer> getPeers(); | ||
33 | + | ||
34 | + /** | ||
35 | + * Returns the actual bgp peer for the given ip address. | ||
36 | + * | ||
37 | + * @param bgpId the id of the bgp peer to fetch | ||
38 | + * @return the interface to this bgp peer | ||
39 | + */ | ||
40 | + BGPPeer getPeer(BGPId bgpId); | ||
41 | + | ||
42 | + /** | ||
28 | * Send a message to a particular bgp peer. | 43 | * Send a message to a particular bgp peer. |
29 | * | 44 | * |
30 | * @param bgpId the id of the peer to send message. | 45 | * @param bgpId the id of the peer to send message. |
... | @@ -41,9 +56,22 @@ public interface BGPController { | ... | @@ -41,9 +56,22 @@ public interface BGPController { |
41 | void processBGPPacket(BGPId bgpId, BGPMessage msg); | 56 | void processBGPPacket(BGPId bgpId, BGPMessage msg); |
42 | 57 | ||
43 | /** | 58 | /** |
59 | + * Close all connected BGP peers. | ||
60 | + * | ||
61 | + */ | ||
62 | + void closeConnectedPeers(); | ||
63 | + | ||
64 | + /** | ||
44 | * Get the BGPConfig class to the caller. | 65 | * Get the BGPConfig class to the caller. |
45 | * | 66 | * |
46 | * @return configuration object | 67 | * @return configuration object |
47 | */ | 68 | */ |
48 | BGPCfg getConfig(); | 69 | BGPCfg getConfig(); |
70 | + | ||
71 | + /** | ||
72 | + * Get the BGP connected peers to this controller. | ||
73 | + * | ||
74 | + * @return the integer number | ||
75 | + */ | ||
76 | + int getBGPConnNumber(); | ||
49 | } | 77 | } |
... | \ No newline at end of file | ... | \ No newline at end of file | ... | ... |
1 | +/* | ||
2 | + * Copyright 2015 Open Networking Laboratory | ||
3 | + * | ||
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | + * you may not use this file except in compliance with the License. | ||
6 | + * You may obtain a copy of the License at | ||
7 | + * | ||
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | + * | ||
10 | + * Unless required by applicable law or agreed to in writing, software | ||
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | + * See the License for the specific language governing permissions and | ||
14 | + * limitations under the License. | ||
15 | + */ | ||
16 | +package org.onosproject.bgp.controller; | ||
17 | +import java.util.List; | ||
18 | +import org.jboss.netty.channel.Channel; | ||
19 | +import org.onosproject.bgpio.protocol.BGPMessage; | ||
20 | +import org.onosproject.bgpio.protocol.BGPVersion; | ||
21 | + | ||
22 | +/** | ||
23 | + * Represents the peer side of an bgp peer. | ||
24 | + * | ||
25 | + */ | ||
26 | +public interface BGPPeer { | ||
27 | + | ||
28 | + /** | ||
29 | + * Sets the BGP version for this bgp peer. | ||
30 | + * | ||
31 | + * @param bgpVersion the version to set. | ||
32 | + */ | ||
33 | + void setBgpPeerVersion(BGPVersion bgpVersion); | ||
34 | + | ||
35 | + /** | ||
36 | + * Gets the BGP version for this bgp peer. | ||
37 | + * | ||
38 | + * @return bgp identifier. | ||
39 | + */ | ||
40 | + int getBgpPeerIdentifier(); | ||
41 | + | ||
42 | + /** | ||
43 | + * Sets the associated Netty channel for this bgp peer. | ||
44 | + * | ||
45 | + * @param channel the Netty channel | ||
46 | + */ | ||
47 | + void setChannel(Channel channel); | ||
48 | + | ||
49 | + /** | ||
50 | + * Gets the associated Netty channel handler for this bgp peer. | ||
51 | + * | ||
52 | + * @return Channel channel connected. | ||
53 | + */ | ||
54 | + Channel getChannel(); | ||
55 | + | ||
56 | + /** | ||
57 | + * Sets the AS Number for this bgp peer. | ||
58 | + * | ||
59 | + * @param peerASNum the autonomous system number value to set. | ||
60 | + */ | ||
61 | + void setBgpPeerASNum(short peerASNum); | ||
62 | + | ||
63 | + /** | ||
64 | + * Sets the hold time for this bgp peer. | ||
65 | + * | ||
66 | + * @param peerHoldTime the hold timer value to set. | ||
67 | + */ | ||
68 | + void setBgpPeerHoldTime(short peerHoldTime); | ||
69 | + | ||
70 | + /** | ||
71 | + * Sets the peer identifier value. | ||
72 | + * | ||
73 | + * @param peerIdentifier the bgp peer identifier value. | ||
74 | + */ | ||
75 | + void setBgpPeerIdentifier(int peerIdentifier); | ||
76 | + | ||
77 | + /** | ||
78 | + * Sets whether the bgp peer is connected. | ||
79 | + * | ||
80 | + * @param connected whether the bgp peer is connected | ||
81 | + */ | ||
82 | + void setConnected(boolean connected); | ||
83 | + | ||
84 | + /** | ||
85 | + * Initialises the behaviour. | ||
86 | + * | ||
87 | + * @param bgpId id of bgp peer | ||
88 | + * @param bgpVersion BGP version | ||
89 | + * @param pktStats packet statistics | ||
90 | + */ | ||
91 | + void init(BGPId bgpId, BGPVersion bgpVersion, BGPPacketStats pktStats); | ||
92 | + | ||
93 | + /** | ||
94 | + * Checks whether the handshake is complete. | ||
95 | + * | ||
96 | + * @return true is finished, false if not. | ||
97 | + */ | ||
98 | + boolean isHandshakeComplete(); | ||
99 | + | ||
100 | + /** | ||
101 | + * Writes the message to the peer. | ||
102 | + * | ||
103 | + * @param msg the message to write | ||
104 | + */ | ||
105 | + void sendMessage(BGPMessage msg); | ||
106 | + | ||
107 | + /** | ||
108 | + * Writes the BGPMessage list to the peer. | ||
109 | + * | ||
110 | + * @param msgs the messages to be written | ||
111 | + */ | ||
112 | + void sendMessage(List<BGPMessage> msgs); | ||
113 | + | ||
114 | + /** | ||
115 | + * Gets a string version of the ID for this bgp peer. | ||
116 | + * | ||
117 | + * @return string version of the ID | ||
118 | + */ | ||
119 | + String getStringId(); | ||
120 | + | ||
121 | + /** | ||
122 | + * Gets the ipAddress of the peer. | ||
123 | + * | ||
124 | + * @return the peer bgpId in IPAddress format | ||
125 | + */ | ||
126 | + BGPId getBGPId(); | ||
127 | + | ||
128 | + /** | ||
129 | + * Checks if the bgp peer is still connected. | ||
130 | + * | ||
131 | + * @return whether the bgp peer is still connected | ||
132 | + */ | ||
133 | + boolean isConnected(); | ||
134 | + | ||
135 | + /** | ||
136 | + * Disconnects the bgp peer by closing the TCP connection. Results in a call to the channel handler's | ||
137 | + * channelDisconnected method for cleanup | ||
138 | + */ | ||
139 | + void disconnectPeer(); | ||
140 | + | ||
141 | + /** | ||
142 | + * Identifies the channel used to communicate with the bgp peer. | ||
143 | + * | ||
144 | + * @return string representation of the connection to the peer | ||
145 | + */ | ||
146 | + String channelId(); | ||
147 | + | ||
148 | + /** | ||
149 | + * Gets the negotiated hold time. | ||
150 | + * | ||
151 | + * @return the negotiated hold time | ||
152 | + */ | ||
153 | + int getNegotiatedHoldTime(); | ||
154 | + | ||
155 | + /** | ||
156 | + * Sets negotiated hold time for the peer. | ||
157 | + * | ||
158 | + * @param negotiatedHoldTime negotiated hold time | ||
159 | + */ | ||
160 | + void setNegotiatedHoldTime(short negotiatedHoldTime); | ||
161 | +} |
... | @@ -61,27 +61,6 @@ public interface BGPMessage extends Writeable { | ... | @@ -61,27 +61,6 @@ public interface BGPMessage extends Writeable { |
61 | BGPMessage build() throws BGPParseException; | 61 | BGPMessage build() throws BGPParseException; |
62 | 62 | ||
63 | /** | 63 | /** |
64 | - * Returns BGP Version of BGP Message. | ||
65 | - * | ||
66 | - * @return BGP Version of BGP Message | ||
67 | - */ | ||
68 | - BGPVersion getVersion(); | ||
69 | - | ||
70 | - /** | ||
71 | - * Returns BGP Type of BGP Message. | ||
72 | - * | ||
73 | - * @return BGP Type of BGP Message | ||
74 | - */ | ||
75 | - BGPType getType(); | ||
76 | - | ||
77 | - /** | ||
78 | - * Returns BGP Header of BGP Message. | ||
79 | - * | ||
80 | - * @return BGP Header of BGP Message | ||
81 | - */ | ||
82 | - BGPHeader getHeader(); | ||
83 | - | ||
84 | - /** | ||
85 | * Sets BgpHeader and return its builder. | 64 | * Sets BgpHeader and return its builder. |
86 | * | 65 | * |
87 | * @param bgpMsgHeader BGP Message Header | 66 | * @param bgpMsgHeader BGP Message Header | ... | ... |
... | @@ -16,19 +16,616 @@ | ... | @@ -16,19 +16,616 @@ |
16 | 16 | ||
17 | package org.onosproject.bgp.controller.impl; | 17 | package org.onosproject.bgp.controller.impl; |
18 | 18 | ||
19 | +import java.io.IOException; | ||
20 | +import java.net.InetSocketAddress; | ||
21 | +import java.net.SocketAddress; | ||
22 | +import java.nio.channels.ClosedChannelException; | ||
23 | +import java.util.Date; | ||
24 | +import java.util.List; | ||
25 | +import java.util.concurrent.RejectedExecutionException; | ||
26 | + | ||
27 | +import org.jboss.netty.channel.Channel; | ||
28 | +import org.jboss.netty.channel.ChannelHandlerContext; | ||
29 | +import org.jboss.netty.channel.ChannelStateEvent; | ||
30 | +import org.jboss.netty.channel.ExceptionEvent; | ||
31 | +import org.jboss.netty.channel.MessageEvent; | ||
19 | import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler; | 32 | import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler; |
33 | +import org.jboss.netty.handler.timeout.ReadTimeoutException; | ||
34 | +import org.jboss.netty.handler.timeout.ReadTimeoutHandler; | ||
35 | +import org.onlab.packet.IpAddress; | ||
36 | +import org.onosproject.bgp.controller.BGPCfg; | ||
37 | +import org.onosproject.bgp.controller.BGPId; | ||
38 | +import org.onosproject.bgp.controller.BGPPeer; | ||
39 | +import org.onosproject.bgp.controller.BGPPeerCfg; | ||
40 | +import org.onosproject.bgp.controller.impl.BGPControllerImpl.BGPPeerManager; | ||
41 | +import org.onosproject.bgpio.exceptions.BGPParseException; | ||
42 | +import org.onosproject.bgpio.protocol.BGPMessage; | ||
43 | +//import org.onosproject.bgpio.protocol.BGPOpenMsg; | ||
44 | +import org.onosproject.bgpio.protocol.BGPType; | ||
45 | +import org.onosproject.bgpio.protocol.BGPVersion; | ||
46 | +import org.slf4j.Logger; | ||
47 | +import org.slf4j.LoggerFactory; | ||
20 | 48 | ||
21 | /** | 49 | /** |
22 | * Channel handler deals with the bgp peer connection and dispatches messages from peer to the appropriate locations. | 50 | * Channel handler deals with the bgp peer connection and dispatches messages from peer to the appropriate locations. |
23 | */ | 51 | */ |
24 | class BGPChannelHandler extends IdleStateAwareChannelHandler { | 52 | class BGPChannelHandler extends IdleStateAwareChannelHandler { |
25 | 53 | ||
26 | - // TODO: implement FSM and session handling mechanism | 54 | + private static final Logger log = LoggerFactory.getLogger(BGPChannelHandler.class); |
55 | + | ||
56 | + static final int BGP_MAX_KEEPALIVE_INTERVAL = 3; | ||
57 | + private BGPPeer bgpPeer; | ||
58 | + private BGPId thisbgpId; | ||
59 | + Channel channel; | ||
60 | + private BGPKeepAliveTimer keepAliveTimer = null; | ||
61 | + private short peerHoldTime = 0; | ||
62 | + private short negotiatedHoldTime = 0; | ||
63 | + private short peerAsNum; | ||
64 | + private int peerIdentifier; | ||
65 | + private BGPPacketStatsImpl bgpPacketStats; | ||
66 | + static final int MAX_WRONG_COUNT_PACKET = 5; | ||
67 | + | ||
68 | + // State needs to be volatile because the HandshakeTimeoutHandler | ||
69 | + // needs to check if the handshake is complete | ||
70 | + private volatile ChannelState state; | ||
71 | + | ||
72 | + // When a bgp peer with a ip addresss is found (i.e we already have a | ||
73 | + // connected peer with the same ip), the new peer is immediately | ||
74 | + // disconnected. At that point netty callsback channelDisconnected() which | ||
75 | + // proceeds to cleaup peer state - we need to ensure that it does not cleanup | ||
76 | + // peer state for the older (still connected) peer | ||
77 | + private volatile Boolean duplicateBGPIdFound; | ||
78 | + // Indicates the bgp version used by this bgp peer | ||
79 | + protected BGPVersion bgpVersion; | ||
80 | + private BGPControllerImpl bgpControllerImpl; | ||
81 | + private BGPPeerManager peerManager; | ||
82 | + private InetSocketAddress inetAddress; | ||
83 | + private IpAddress ipAddress; | ||
84 | + private SocketAddress address; | ||
85 | + private String peerAddr; | ||
86 | + private BGPCfg bgpconfig; | ||
87 | + | ||
27 | /** | 88 | /** |
28 | * Create a new unconnected BGPChannelHandler. | 89 | * Create a new unconnected BGPChannelHandler. |
29 | * | 90 | * |
30 | * @param bgpCtrlImpl bgp controller implementation object | 91 | * @param bgpCtrlImpl bgp controller implementation object |
31 | */ | 92 | */ |
32 | BGPChannelHandler(BGPControllerImpl bgpCtrlImpl) { | 93 | BGPChannelHandler(BGPControllerImpl bgpCtrlImpl) { |
94 | + this.bgpControllerImpl = bgpCtrlImpl; | ||
95 | + this.peerManager = bgpCtrlImpl.getPeerManager(); | ||
96 | + this.state = ChannelState.IDLE; | ||
97 | + this.duplicateBGPIdFound = Boolean.FALSE; | ||
98 | + this.bgpPacketStats = new BGPPacketStatsImpl(); | ||
99 | + this.bgpconfig = bgpCtrlImpl.getConfig(); | ||
100 | + } | ||
101 | + | ||
102 | + // To disconnect peer session. | ||
103 | + public void disconnectPeer() { | ||
104 | + bgpPeer.disconnectPeer(); | ||
105 | + } | ||
106 | + | ||
107 | + // ************************* | ||
108 | + // Channel State Machine | ||
109 | + // ************************* | ||
110 | + | ||
111 | + /** | ||
112 | + * The state machine for handling the peer/channel state. All state transitions should happen from within the state | ||
113 | + * machine (and not from other parts of the code) | ||
114 | + */ | ||
115 | + enum ChannelState { | ||
116 | + /** | ||
117 | + * Initial state before channel is connected. | ||
118 | + */ | ||
119 | + IDLE(false) { | ||
120 | + | ||
121 | + }, | ||
122 | + | ||
123 | + OPENSENT(false) { | ||
124 | + @Override | ||
125 | + void processBGPMessage(BGPChannelHandler h, BGPMessage m) throws IOException, BGPParseException { | ||
126 | + log.debug("message received in OPENSENT state"); | ||
127 | + // check for OPEN message | ||
128 | + if (m.getType() != BGPType.OPEN) { | ||
129 | + // When the message type is not keep alive message increment the wrong packet statistics | ||
130 | + h.processUnknownMsg(); | ||
131 | + log.debug("Message is not OPEN message"); | ||
132 | + } else { | ||
133 | + log.debug("Sending keep alive message in OPENSENT state"); | ||
134 | + h.bgpPacketStats.addInPacket(); | ||
135 | + | ||
136 | + // TODO: initialize openmessage BGPOpenMsg pOpenmsg = (BGPOpenMsg) m; | ||
137 | + // TODO: initialize identifier from open messgae h.peerIdentifier = pOpenmsg.getBgpId(); | ||
138 | + | ||
139 | + // validate capabilities and open msg | ||
140 | + if (h.openMsgValidation(h)) { | ||
141 | + log.debug("Sending handshake OPEN message"); | ||
142 | + | ||
143 | + /* | ||
144 | + * RFC 4271, section 4.2: Upon receipt of an OPEN message, a BGP speaker MUST calculate the | ||
145 | + * value of the Hold Timer by using the smaller of its configured Hold Time and the Hold Time | ||
146 | + * received in the OPEN message | ||
147 | + */ | ||
148 | + // TODO: initialize holdtime from open message h.peerHoldTime = pOpenmsg.getHoldTime(); | ||
149 | + if (h.peerHoldTime < h.bgpconfig.getHoldTime()) { | ||
150 | + h.channel.getPipeline().replace("holdTime", | ||
151 | + "holdTime", | ||
152 | + new ReadTimeoutHandler(BGPPipelineFactory.TIMER, | ||
153 | + h.peerHoldTime)); | ||
154 | + } | ||
155 | + | ||
156 | + log.info("Hold Time : " + h.peerHoldTime); | ||
157 | + | ||
158 | + // TODO: get AS number for open message update AS number | ||
159 | + } | ||
160 | + | ||
161 | + // Send keepalive message to peer. | ||
162 | + h.sendKeepAliveMessage(); | ||
163 | + h.bgpPacketStats.addOutPacket(); | ||
164 | + h.setState(OPENCONFIRM); | ||
165 | + h.bgpconfig.setPeerConnState(h.peerAddr, BGPPeerCfg.State.OPENCONFIRM); | ||
166 | + } | ||
167 | + } | ||
168 | + }, | ||
169 | + | ||
170 | + OPENWAIT(false) { | ||
171 | + @Override | ||
172 | + void processBGPMessage(BGPChannelHandler h, BGPMessage m) throws IOException, BGPParseException { | ||
173 | + log.debug("Message received in OPEN WAIT State"); | ||
174 | + | ||
175 | + // check for open message | ||
176 | + if (m.getType() != BGPType.OPEN) { | ||
177 | + // When the message type is not open message increment the wrong packet statistics | ||
178 | + h.processUnknownMsg(); | ||
179 | + log.debug("Message is not OPEN message"); | ||
180 | + } else { | ||
181 | + h.bgpPacketStats.addInPacket(); | ||
182 | + | ||
183 | + // TODO: initialize open message BGPOpenMsg pOpenmsg = (BGPOpenMsg) m; | ||
184 | + | ||
185 | + // Validate open message | ||
186 | + if (h.openMsgValidation(h)) { | ||
187 | + log.debug("Sending handshake OPEN message"); | ||
188 | + | ||
189 | + /* | ||
190 | + * RFC 4271, section 4.2: Upon receipt of an OPEN message, a BGP speaker MUST calculate the | ||
191 | + * value of the Hold Timer by using the smaller of its configured Hold Time and the Hold Time | ||
192 | + * received in the OPEN message | ||
193 | + */ | ||
194 | + // TODO: get hold time from open message h.peerHoldTime = pOpenmsg.getHoldTime(); | ||
195 | + if (h.peerHoldTime < h.bgpconfig.getHoldTime()) { | ||
196 | + h.channel.getPipeline().replace("holdTime", | ||
197 | + "holdTime", | ||
198 | + new ReadTimeoutHandler(BGPPipelineFactory.TIMER, | ||
199 | + h.peerHoldTime)); | ||
200 | + } | ||
201 | + | ||
202 | + log.debug("Hold Time : " + h.peerHoldTime); | ||
203 | + | ||
204 | + //TODO: update AS number form open messsage update AS number | ||
205 | + | ||
206 | + h.sendHandshakeOpenMessage(); | ||
207 | + h.bgpPacketStats.addOutPacket(); | ||
208 | + h.setState(OPENCONFIRM); | ||
209 | + } | ||
210 | + } | ||
211 | + } | ||
212 | + }, | ||
213 | + | ||
214 | + OPENCONFIRM(false) { | ||
215 | + @Override | ||
216 | + void processBGPMessage(BGPChannelHandler h, BGPMessage m) throws IOException, BGPParseException { | ||
217 | + log.debug("Message received in OPENCONFIRM state"); | ||
218 | + // check for keep alive message | ||
219 | + if (m.getType() != BGPType.KEEP_ALIVE) { | ||
220 | + // When the message type is not keep alive message handle the wrong packet | ||
221 | + h.processUnknownMsg(); | ||
222 | + log.debug("Message is not KEEPALIVE message"); | ||
223 | + } else { | ||
224 | + | ||
225 | + // Set the peer connected status | ||
226 | + h.bgpPacketStats.addInPacket(); | ||
227 | + log.debug("Sending keep alive message in OPENCONFIRM state"); | ||
228 | + | ||
229 | + final InetSocketAddress inetAddress = (InetSocketAddress) h.address; | ||
230 | + h.thisbgpId = BGPId.bgpId(IpAddress.valueOf(inetAddress.getAddress())); | ||
231 | + | ||
232 | + h.bgpPeer = h.peerManager.getBGPPeerInstance(h.thisbgpId, h.bgpVersion, h.bgpPacketStats); | ||
233 | + // set the status fo bgp as connected | ||
234 | + h.bgpPeer.setConnected(true); | ||
235 | + h.bgpPeer.setChannel(h.channel); | ||
236 | + | ||
237 | + // set specific parameters to bgp peer | ||
238 | + h.bgpPeer.setBgpPeerVersion(h.bgpVersion); | ||
239 | + h.bgpPeer.setBgpPeerASNum(h.peerAsNum); | ||
240 | + h.bgpPeer.setBgpPeerHoldTime(h.peerHoldTime); | ||
241 | + h.bgpPeer.setBgpPeerIdentifier(h.peerIdentifier); | ||
242 | + | ||
243 | + h.negotiatedHoldTime = (h.peerHoldTime < h.bgpconfig.getHoldTime()) ? h.peerHoldTime : h.bgpconfig | ||
244 | + .getHoldTime(); | ||
245 | + h.bgpPeer.setNegotiatedHoldTime(h.negotiatedHoldTime); | ||
246 | + /* | ||
247 | + * RFC 4271, When an OPEN message is received, sends a KEEPALIVE message, If the negotiated hold | ||
248 | + * time value is zero, then the HoldTimer and KeepaliveTimer are not started. A reasonable maximum | ||
249 | + * time between KEEPALIVE messages would be one third of the Hold Time interval. | ||
250 | + */ | ||
251 | + h.sendKeepAliveMessage(); | ||
252 | + | ||
253 | + if (h.negotiatedHoldTime != 0) { | ||
254 | + h.keepAliveTimer | ||
255 | + = new BGPKeepAliveTimer(h, (h.negotiatedHoldTime / BGP_MAX_KEEPALIVE_INTERVAL)); | ||
256 | + } | ||
257 | + | ||
258 | + h.bgpPacketStats.addOutPacket(); | ||
259 | + | ||
260 | + // set the state handshake completion. | ||
261 | + h.setHandshakeComplete(true); | ||
262 | + | ||
263 | + if (!h.peerManager.addConnectedPeer(h.thisbgpId, h.bgpPeer)) { | ||
264 | + /* | ||
265 | + * RFC 4271, Section 6.8, Based on the value of the BGP identifier, a convention is established | ||
266 | + * for detecting which BGP connection is to be preserved when a collision occurs. The convention | ||
267 | + * is to compare the BGP Identifiers of the peers involved in the collision and to retain only | ||
268 | + * the connection initiated by the BGP speaker with the higher-valued BGP Identifier.. | ||
269 | + */ | ||
270 | + // TODO: Connection collision handling. | ||
271 | + disconnectDuplicate(h); | ||
272 | + } else { | ||
273 | + h.setState(ESTABLISHED); | ||
274 | + h.bgpconfig.setPeerConnState(h.peerAddr, BGPPeerCfg.State.ESTABLISHED); | ||
275 | + } | ||
276 | + } | ||
277 | + } | ||
278 | + }, | ||
279 | + | ||
280 | + ESTABLISHED(true) { | ||
281 | + @Override | ||
282 | + void processBGPMessage(BGPChannelHandler h, BGPMessage m) throws IOException, BGPParseException { | ||
283 | + log.debug("Message received in established state " + m.getType()); | ||
284 | + // dispatch the message | ||
285 | + h.dispatchMessage(m); | ||
286 | + } | ||
287 | + }; | ||
288 | + | ||
289 | + private boolean handshakeComplete; | ||
290 | + | ||
291 | + ChannelState(boolean handshakeComplete) { | ||
292 | + this.handshakeComplete = handshakeComplete; | ||
293 | + } | ||
294 | + | ||
295 | + /** | ||
296 | + * Is this a state in which the handshake has completed? | ||
297 | + * | ||
298 | + * @return true if the handshake is complete | ||
299 | + */ | ||
300 | + public boolean isHandshakeComplete() { | ||
301 | + return this.handshakeComplete; | ||
302 | + } | ||
303 | + | ||
304 | + /** | ||
305 | + * Disconnect duplicate peer connection. | ||
306 | + * | ||
307 | + * @param h channel handler | ||
308 | + */ | ||
309 | + protected void disconnectDuplicate(BGPChannelHandler h) { | ||
310 | + log.error("Duplicated BGP IP or incompleted cleanup - " + "" + "disconnecting channel {}", | ||
311 | + h.getPeerInfoString()); | ||
312 | + h.duplicateBGPIdFound = Boolean.TRUE; | ||
313 | + h.channel.disconnect(); | ||
314 | + } | ||
315 | + | ||
316 | + // set handshake completion status | ||
317 | + public void setHandshakeComplete(boolean handshakeComplete) { | ||
318 | + this.handshakeComplete = handshakeComplete; | ||
319 | + } | ||
320 | + | ||
321 | + void processBGPMessage(BGPChannelHandler bgpChannelHandler, BGPMessage pm) | ||
322 | + throws IOException, BGPParseException { | ||
323 | + // TODO Auto-generated method stub | ||
324 | + log.debug("BGP message stub"); | ||
325 | + } | ||
326 | + | ||
327 | + } | ||
328 | + | ||
329 | + // ************************* | ||
330 | + // Channel handler methods | ||
331 | + // ************************* | ||
332 | + | ||
333 | + @Override | ||
334 | + public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception { | ||
335 | + | ||
336 | + channel = e.getChannel(); | ||
337 | + log.info("BGP connected from {}", channel.getRemoteAddress()); | ||
338 | + | ||
339 | + address = channel.getRemoteAddress(); | ||
340 | + if (!(address instanceof InetSocketAddress)) { | ||
341 | + throw new IOException("Invalid peer connection."); | ||
342 | + } | ||
343 | + | ||
344 | + // Connection should establish only if local ip and Autonomous system number is configured. | ||
345 | + if (bgpconfig.getState() != BGPCfg.State.IP_AS_CONFIGURED) { | ||
346 | + channel.close(); | ||
347 | + log.info("BGP local AS and router ID not configured"); | ||
348 | + return; | ||
349 | + } | ||
350 | + | ||
351 | + inetAddress = (InetSocketAddress) address; | ||
352 | + ipAddress = IpAddress.valueOf(inetAddress.getAddress()); | ||
353 | + peerAddr = ipAddress.toString(); | ||
354 | + | ||
355 | + // if peer is not configured disconnect session | ||
356 | + if (!bgpconfig.isPeerConfigured(peerAddr)) { | ||
357 | + log.debug("Peer is not configured {}", peerAddr); | ||
358 | + channel.close(); | ||
359 | + return; | ||
360 | + } | ||
361 | + | ||
362 | + // if connection is already established close channel | ||
363 | + if (peerManager.isPeerConnected(peerAddr)) { | ||
364 | + log.debug("Duplicate connection received, peer {}", peerAddr); | ||
365 | + channel.close(); | ||
366 | + return; | ||
367 | + } | ||
368 | + | ||
369 | + if (null != channel.getPipeline().get("PassiveHandler")) { | ||
370 | + log.info("BGP handle connection request from peer"); | ||
371 | + // Wait for open message from bgp peer | ||
372 | + setState(ChannelState.OPENWAIT); | ||
373 | + } else if (null != channel.getPipeline().get("ActiveHandler")) { | ||
374 | + log.info("BGP handle connection response from peer"); | ||
375 | + | ||
376 | + sendHandshakeOpenMessage(); | ||
377 | + bgpPacketStats.addOutPacket(); | ||
378 | + setState(ChannelState.OPENSENT); | ||
379 | + bgpconfig.setPeerConnState(peerAddr, BGPPeerCfg.State.OPENSENT); | ||
380 | + } | ||
381 | + } | ||
382 | + | ||
383 | + @Override | ||
384 | + public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception { | ||
385 | + | ||
386 | + channel = e.getChannel(); | ||
387 | + log.info("BGP disconnected callback for bgp:{}. Cleaning up ...", getPeerInfoString()); | ||
388 | + | ||
389 | + address = channel.getRemoteAddress(); | ||
390 | + if (!(address instanceof InetSocketAddress)) { | ||
391 | + throw new IOException("Invalid peer connection."); | ||
392 | + } | ||
393 | + | ||
394 | + inetAddress = (InetSocketAddress) address; | ||
395 | + ipAddress = IpAddress.valueOf(inetAddress.getAddress()); | ||
396 | + peerAddr = ipAddress.toString(); | ||
397 | + | ||
398 | + if (thisbgpId != null) { | ||
399 | + if (!duplicateBGPIdFound) { | ||
400 | + // if the disconnected peer (on this ChannelHandler) | ||
401 | + // was not one with a duplicate, it is safe to remove all | ||
402 | + // state for it at the controller. Notice that if the disconnected | ||
403 | + // peer was a duplicate-ip, calling the method below would clear | ||
404 | + // all state for the original peer (with the same ip), | ||
405 | + // which we obviously don't want. | ||
406 | + log.debug("{}:removal called", getPeerInfoString()); | ||
407 | + if (bgpPeer != null) { | ||
408 | + peerManager.removeConnectedPeer(thisbgpId); | ||
409 | + } | ||
410 | + } else { | ||
411 | + // A duplicate was disconnected on this ChannelHandler, | ||
412 | + // this is the same peer reconnecting, but the original state was | ||
413 | + // not cleaned up - XXX check liveness of original ChannelHandler | ||
414 | + log.debug("{}:duplicate found", getPeerInfoString()); | ||
415 | + duplicateBGPIdFound = Boolean.FALSE; | ||
416 | + } | ||
417 | + | ||
418 | + if (null != keepAliveTimer) { | ||
419 | + keepAliveTimer.getKeepAliveTimer().cancel(); | ||
420 | + } | ||
421 | + } else { | ||
422 | + log.warn("No bgp ip in channelHandler registered for " + "disconnected peer {}", getPeerInfoString()); | ||
423 | + } | ||
424 | + } | ||
425 | + | ||
426 | + @Override | ||
427 | + public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception { | ||
428 | + | ||
429 | + log.info("[exceptionCaught]: " + e.toString()); | ||
430 | + | ||
431 | + if (e.getCause() instanceof ReadTimeoutException) { | ||
432 | + if ((ChannelState.OPENWAIT == state) || (ChannelState.OPENSENT == state)) { | ||
433 | + | ||
434 | + // When ReadTimeout timer is expired in OPENWAIT/OPENSENT state, it is considered | ||
435 | + // TODO: Send notification | ||
436 | + channel.close(); | ||
437 | + state = ChannelState.IDLE; | ||
438 | + return; | ||
439 | + } else if (ChannelState.OPENCONFIRM == state) { | ||
440 | + | ||
441 | + // When ReadTimeout timer is expired in OPENCONFIRM state. | ||
442 | + // TODO: Send Notification | ||
443 | + channel.close(); | ||
444 | + state = ChannelState.IDLE; | ||
445 | + return; | ||
446 | + } | ||
447 | + } else if (e.getCause() instanceof ClosedChannelException) { | ||
448 | + log.debug("Channel for bgp {} already closed", getPeerInfoString()); | ||
449 | + } else if (e.getCause() instanceof IOException) { | ||
450 | + log.error("Disconnecting peer {} due to IO Error: {}", getPeerInfoString(), e.getCause().getMessage()); | ||
451 | + if (log.isDebugEnabled()) { | ||
452 | + // still print stack trace if debug is enabled | ||
453 | + log.debug("StackTrace for previous Exception: ", e.getCause()); | ||
454 | + } | ||
455 | + channel.close(); | ||
456 | + } else if (e.getCause() instanceof BGPParseException) { | ||
457 | + // TODO: SEND NOTIFICATION | ||
458 | + log.debug("BGP Parse Exception: ", e.getCause()); | ||
459 | + } else if (e.getCause() instanceof RejectedExecutionException) { | ||
460 | + log.warn("Could not process message: queue full"); | ||
461 | + } else { | ||
462 | + log.error("Error while processing message from peer " + getPeerInfoString() + "state " + this.state); | ||
463 | + channel.close(); | ||
464 | + } | ||
465 | + } | ||
466 | + | ||
467 | + @Override | ||
468 | + public String toString() { | ||
469 | + return getPeerInfoString(); | ||
470 | + } | ||
471 | + | ||
472 | + @Override | ||
473 | + public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { | ||
474 | + if (e.getMessage() instanceof List) { | ||
475 | + @SuppressWarnings("Unchecked") | ||
476 | + List<BGPMessage> msglist = (List<BGPMessage>) e.getMessage(); | ||
477 | + for (BGPMessage pm : msglist) { | ||
478 | + // Do the actual packet processing | ||
479 | + state.processBGPMessage(this, pm); | ||
480 | + } | ||
481 | + } else { | ||
482 | + state.processBGPMessage(this, (BGPMessage) e.getMessage()); | ||
483 | + } | ||
484 | + } | ||
485 | + | ||
486 | + // ************************* | ||
487 | + // Channel utility methods | ||
488 | + // ************************* | ||
489 | + /** | ||
490 | + * Set handshake status. | ||
491 | + * | ||
492 | + * @param handshakeComplete handshake complete status | ||
493 | + */ | ||
494 | + public void setHandshakeComplete(boolean handshakeComplete) { | ||
495 | + this.state.setHandshakeComplete(handshakeComplete); | ||
496 | + } | ||
497 | + | ||
498 | + /** | ||
499 | + * Is this a state in which the handshake has completed? | ||
500 | + * | ||
501 | + * @return true if the handshake is complete | ||
502 | + */ | ||
503 | + public boolean isHandshakeComplete() { | ||
504 | + return state.isHandshakeComplete(); | ||
505 | + } | ||
506 | + | ||
507 | + /** | ||
508 | + * To handle the BGP message. | ||
509 | + * | ||
510 | + * @param m BGP message | ||
511 | + */ | ||
512 | + private void dispatchMessage(BGPMessage m) throws BGPParseException { | ||
513 | + bgpPacketStats.addInPacket(); | ||
514 | + bgpControllerImpl.processBGPPacket(thisbgpId, m); | ||
515 | + } | ||
516 | + | ||
517 | + /** | ||
518 | + * Return a string describing this peer based on the already available information (ip address and/or remote | ||
519 | + * socket). | ||
520 | + * | ||
521 | + * @return display string | ||
522 | + */ | ||
523 | + private String getPeerInfoString() { | ||
524 | + if (bgpPeer != null) { | ||
525 | + return bgpPeer.toString(); | ||
526 | + } | ||
527 | + String channelString; | ||
528 | + if (channel == null || channel.getRemoteAddress() == null) { | ||
529 | + channelString = "?"; | ||
530 | + } else { | ||
531 | + channelString = channel.getRemoteAddress().toString(); | ||
532 | + } | ||
533 | + String bgpIpString; | ||
534 | + // TODO: implement functionality to get bgp id string | ||
535 | + bgpIpString = "?"; | ||
536 | + return String.format("[%s BGP-IP[%s]]", channelString, bgpIpString); | ||
537 | + } | ||
538 | + | ||
539 | + /** | ||
540 | + * Update the channels state. Only called from the state machine. TODO: enforce restricted state transitions | ||
541 | + * | ||
542 | + * @param state | ||
543 | + */ | ||
544 | + private void setState(ChannelState state) { | ||
545 | + this.state = state; | ||
546 | + } | ||
547 | + | ||
548 | + /** | ||
549 | + * get packet statistics. | ||
550 | + * | ||
551 | + * @return packet statistics | ||
552 | + */ | ||
553 | + public BGPPacketStatsImpl getBgpPacketStats() { | ||
554 | + return bgpPacketStats; | ||
555 | + } | ||
556 | + | ||
557 | + /** | ||
558 | + * Send handshake open message to the peer. | ||
559 | + * | ||
560 | + * @throws IOException ,BGPParseException | ||
561 | + */ | ||
562 | + private void sendHandshakeOpenMessage() throws IOException, BGPParseException { | ||
563 | + // TODO: send open message. | ||
564 | + | ||
565 | + } | ||
566 | + | ||
567 | + /** | ||
568 | + * Send keep alive message. | ||
569 | + * | ||
570 | + * @throws IOException when channel is disconnected | ||
571 | + * @throws BGPParseException while building keep alive message | ||
572 | + */ | ||
573 | + synchronized void sendKeepAliveMessage() throws IOException, BGPParseException { | ||
574 | + | ||
575 | + // TODO: send keep alive message. | ||
576 | + } | ||
577 | + | ||
578 | + /** | ||
579 | + * Send notification and close channel with peer. | ||
580 | + */ | ||
581 | + private void sendErrNotificationAndCloseChannel() { | ||
582 | + // TODO: send notification | ||
583 | + channel.close(); | ||
584 | + } | ||
585 | + | ||
586 | + /** | ||
587 | + * Process unknown BGP message received. | ||
588 | + * | ||
589 | + * @throws BGPParseException when received invalid message | ||
590 | + */ | ||
591 | + public void processUnknownMsg() throws BGPParseException { | ||
592 | + log.debug("UNKNOWN message received"); | ||
593 | + Date now = null; | ||
594 | + if (bgpPacketStats.wrongPacketCount() == 0) { | ||
595 | + now = new Date(); | ||
596 | + bgpPacketStats.setTime(now.getTime()); | ||
597 | + bgpPacketStats.addWrongPacket(); | ||
598 | + sendErrNotificationAndCloseChannel(); | ||
599 | + } | ||
600 | + if (bgpPacketStats.wrongPacketCount() > 1) { | ||
601 | + Date lastest = new Date(); | ||
602 | + bgpPacketStats.addWrongPacket(); | ||
603 | + // converting to seconds | ||
604 | + if (((lastest.getTime() - bgpPacketStats.getTime()) / 1000) > 60) { | ||
605 | + now = lastest; | ||
606 | + bgpPacketStats.setTime(now.getTime()); | ||
607 | + bgpPacketStats.resetWrongPacket(); | ||
608 | + bgpPacketStats.addWrongPacket(); | ||
609 | + } else if (((int) (lastest.getTime() - now.getTime()) / 1000) < 60) { | ||
610 | + if (MAX_WRONG_COUNT_PACKET <= bgpPacketStats.wrongPacketCount()) { | ||
611 | + // reset once wrong packet count reaches MAX_WRONG_COUNT_PACKET | ||
612 | + bgpPacketStats.resetWrongPacket(); | ||
613 | + // max wrong packets received send error message and close the session | ||
614 | + sendErrNotificationAndCloseChannel(); | ||
615 | + } | ||
616 | + } | ||
617 | + } | ||
618 | + } | ||
619 | + | ||
620 | + /** | ||
621 | + * Open message validation. | ||
622 | + * | ||
623 | + * @param h channel handler | ||
624 | + * @return true if validation succeed, otherwise false | ||
625 | + * @throws BGPParseException when received invalid message | ||
626 | + */ | ||
627 | + public boolean openMsgValidation(BGPChannelHandler h) throws BGPParseException { | ||
628 | + // TODO: Open message validation. | ||
629 | + return true; | ||
33 | } | 630 | } |
34 | } | 631 | } | ... | ... |
... | @@ -17,16 +17,24 @@ | ... | @@ -17,16 +17,24 @@ |
17 | package org.onosproject.bgp.controller.impl; | 17 | package org.onosproject.bgp.controller.impl; |
18 | 18 | ||
19 | import static org.onlab.util.Tools.groupedThreads; | 19 | import static org.onlab.util.Tools.groupedThreads; |
20 | + | ||
21 | +import java.util.concurrent.ConcurrentHashMap; | ||
20 | import java.util.concurrent.ExecutorService; | 22 | import java.util.concurrent.ExecutorService; |
21 | import java.util.concurrent.Executors; | 23 | import java.util.concurrent.Executors; |
24 | +import java.util.concurrent.locks.Lock; | ||
25 | +import java.util.concurrent.locks.ReentrantLock; | ||
22 | import org.apache.felix.scr.annotations.Activate; | 26 | import org.apache.felix.scr.annotations.Activate; |
23 | import org.apache.felix.scr.annotations.Component; | 27 | import org.apache.felix.scr.annotations.Component; |
24 | import org.apache.felix.scr.annotations.Deactivate; | 28 | import org.apache.felix.scr.annotations.Deactivate; |
25 | import org.apache.felix.scr.annotations.Service; | 29 | import org.apache.felix.scr.annotations.Service; |
30 | +import org.onlab.packet.IpAddress; | ||
26 | import org.onosproject.bgp.controller.BGPCfg; | 31 | import org.onosproject.bgp.controller.BGPCfg; |
27 | import org.onosproject.bgp.controller.BGPController; | 32 | import org.onosproject.bgp.controller.BGPController; |
28 | import org.onosproject.bgp.controller.BGPId; | 33 | import org.onosproject.bgp.controller.BGPId; |
34 | +import org.onosproject.bgp.controller.BGPPacketStats; | ||
35 | +import org.onosproject.bgp.controller.BGPPeer; | ||
29 | import org.onosproject.bgpio.protocol.BGPMessage; | 36 | import org.onosproject.bgpio.protocol.BGPMessage; |
37 | +import org.onosproject.bgpio.protocol.BGPVersion; | ||
30 | import org.slf4j.Logger; | 38 | import org.slf4j.Logger; |
31 | import org.slf4j.LoggerFactory; | 39 | import org.slf4j.LoggerFactory; |
32 | 40 | ||
... | @@ -43,7 +51,9 @@ public class BGPControllerImpl implements BGPController { | ... | @@ -43,7 +51,9 @@ public class BGPControllerImpl implements BGPController { |
43 | private final ExecutorService executorBarrier = Executors.newFixedThreadPool(4, | 51 | private final ExecutorService executorBarrier = Executors.newFixedThreadPool(4, |
44 | groupedThreads("onos/bgp", | 52 | groupedThreads("onos/bgp", |
45 | "event-barrier-%d")); | 53 | "event-barrier-%d")); |
54 | + protected ConcurrentHashMap<BGPId, BGPPeer> connectedPeers = new ConcurrentHashMap<BGPId, BGPPeer>(); | ||
46 | 55 | ||
56 | + protected BGPPeerManager peerManager = new BGPPeerManager(); | ||
47 | final Controller ctrl = new Controller(this); | 57 | final Controller ctrl = new Controller(this); |
48 | 58 | ||
49 | private BGPConfig bgpconfig = new BGPConfig(); | 59 | private BGPConfig bgpconfig = new BGPConfig(); |
... | @@ -57,11 +67,22 @@ public class BGPControllerImpl implements BGPController { | ... | @@ -57,11 +67,22 @@ public class BGPControllerImpl implements BGPController { |
57 | @Deactivate | 67 | @Deactivate |
58 | public void deactivate() { | 68 | public void deactivate() { |
59 | // Close all connected peers | 69 | // Close all connected peers |
70 | + closeConnectedPeers(); | ||
60 | this.ctrl.stop(); | 71 | this.ctrl.stop(); |
61 | log.info("Stopped"); | 72 | log.info("Stopped"); |
62 | } | 73 | } |
63 | 74 | ||
64 | @Override | 75 | @Override |
76 | + public Iterable<BGPPeer> getPeers() { | ||
77 | + return this.connectedPeers.values(); | ||
78 | + } | ||
79 | + | ||
80 | + @Override | ||
81 | + public BGPPeer getPeer(BGPId bgpId) { | ||
82 | + return this.connectedPeers.get(bgpId); | ||
83 | + } | ||
84 | + | ||
85 | + @Override | ||
65 | public void writeMsg(BGPId bgpId, BGPMessage msg) { | 86 | public void writeMsg(BGPId bgpId, BGPMessage msg) { |
66 | // TODO: Send message | 87 | // TODO: Send message |
67 | } | 88 | } |
... | @@ -88,17 +109,167 @@ public class BGPControllerImpl implements BGPController { | ... | @@ -88,17 +109,167 @@ public class BGPControllerImpl implements BGPController { |
88 | } | 109 | } |
89 | } | 110 | } |
90 | 111 | ||
112 | + @Override | ||
113 | + public void closeConnectedPeers() { | ||
114 | + BGPPeer bgpPeer; | ||
115 | + for (BGPId id : this.connectedPeers.keySet()) { | ||
116 | + bgpPeer = getPeer(id); | ||
117 | + bgpPeer.disconnectPeer(); | ||
118 | + } | ||
119 | + } | ||
120 | + | ||
121 | + /** | ||
122 | + * Implementation of an BGP Peer which is responsible for keeping track of connected peers and the state in which | ||
123 | + * they are. | ||
124 | + */ | ||
125 | + public class BGPPeerManager { | ||
126 | + | ||
127 | + private final Logger log = LoggerFactory.getLogger(BGPPeerManager.class); | ||
128 | + private final Lock peerLock = new ReentrantLock(); | ||
129 | + | ||
130 | + /** | ||
131 | + * Add a BGP peer that has just connected to the system. | ||
132 | + * | ||
133 | + * @param bgpId the id of bgp peer to add | ||
134 | + * @param bgpPeer the actual bgp peer object. | ||
135 | + * @return true if added, false otherwise. | ||
136 | + */ | ||
137 | + public boolean addConnectedPeer(BGPId bgpId, BGPPeer bgpPeer) { | ||
138 | + | ||
139 | + if (connectedPeers.get(bgpId) != null) { | ||
140 | + this.log.error("Trying to add connectedPeer but found previous " + "value for bgp ip: {}", | ||
141 | + bgpId.toString()); | ||
142 | + return false; | ||
143 | + } else { | ||
144 | + this.log.debug("Added Peer {}", bgpId.toString()); | ||
145 | + connectedPeers.put(bgpId, bgpPeer); | ||
146 | + return true; | ||
147 | + } | ||
148 | + } | ||
149 | + | ||
150 | + /** | ||
151 | + * Checks if the activation for this bgp peer is valid. | ||
152 | + * | ||
153 | + * @param bgpId the id of bgp peer to check | ||
154 | + * @return true if valid, false otherwise | ||
155 | + */ | ||
156 | + public boolean isPeerConnected(BGPId bgpId) { | ||
157 | + if (connectedPeers.get(bgpId) == null) { | ||
158 | + this.log.error("Trying to activate peer but is not in " + "connected peer: bgpIp {}. Aborting ..", | ||
159 | + bgpId.toString()); | ||
160 | + return false; | ||
161 | + } | ||
162 | + | ||
163 | + return true; | ||
164 | + } | ||
165 | + | ||
166 | + /** | ||
167 | + * Checks if the activation for this bgp peer is valid. | ||
168 | + * | ||
169 | + * @param routerid the routerid of bgp peer to check | ||
170 | + * @return true if valid, false otherwise | ||
171 | + */ | ||
172 | + public boolean isPeerConnected(String routerid) { | ||
173 | + | ||
174 | + final BGPId bgpId; | ||
175 | + bgpId = BGPId.bgpId(IpAddress.valueOf(routerid)); | ||
176 | + | ||
177 | + if (connectedPeers.get(bgpId) != null) { | ||
178 | + this.log.info("Peer connection exist "); | ||
179 | + return true; | ||
180 | + } | ||
181 | + this.log.info("Initiate connect request to " + "peer: bgpIp {}", bgpId.toString()); | ||
182 | + | ||
183 | + return false; | ||
184 | + } | ||
185 | + | ||
186 | + /** | ||
187 | + * Clear all state in controller peer maps for a bgp peer that has | ||
188 | + * disconnected from the local controller. | ||
189 | + * | ||
190 | + * @param bgpId the id of bgp peer to remove. | ||
191 | + */ | ||
192 | + public void removeConnectedPeer(BGPId bgpId) { | ||
193 | + connectedPeers.remove(bgpId); | ||
194 | + } | ||
195 | + | ||
196 | + /** | ||
197 | + * Clear all state in controller peer maps for a bgp peer that has | ||
198 | + * disconnected from the local controller. | ||
199 | + * | ||
200 | + * @param routerid the router id of bgp peer to remove. | ||
201 | + */ | ||
202 | + public void removeConnectedPeer(String routerid) { | ||
203 | + final BGPId bgpId; | ||
204 | + | ||
205 | + bgpId = BGPId.bgpId(IpAddress.valueOf(routerid)); | ||
206 | + | ||
207 | + connectedPeers.remove(bgpId); | ||
208 | + } | ||
209 | + | ||
210 | + /** | ||
211 | + * Gets bgp peer for connected peer map. | ||
212 | + * | ||
213 | + * @param routerid router id | ||
214 | + * @return peer if available, null otherwise | ||
215 | + */ | ||
216 | + public BGPPeer getPeer(String routerid) { | ||
217 | + final BGPId bgpId; | ||
218 | + bgpId = BGPId.bgpId(IpAddress.valueOf(routerid)); | ||
219 | + | ||
220 | + return connectedPeers.get(bgpId); | ||
221 | + } | ||
222 | + | ||
223 | + /** | ||
224 | + * Gets bgp peer instance. | ||
225 | + * | ||
226 | + * @param bgpId bgp identifier. | ||
227 | + * @param pv bgp version. | ||
228 | + * @param pktStats packet statistics. | ||
229 | + * @return BGPPeer peer instance. | ||
230 | + */ | ||
231 | + public BGPPeer getBGPPeerInstance(BGPId bgpId, BGPVersion pv, BGPPacketStats pktStats) { | ||
232 | + BGPPeer bgpPeer = new BGPPeerImpl(); | ||
233 | + bgpPeer.init(bgpId, pv, pktStats); | ||
234 | + return bgpPeer; | ||
235 | + } | ||
236 | + | ||
237 | + } | ||
238 | + | ||
91 | /** | 239 | /** |
92 | - * Get controller instance. | 240 | + * Gets controller instance. |
93 | * | 241 | * |
94 | - * @return ctrl the controller. | 242 | + * @return Controller instance. |
95 | */ | 243 | */ |
96 | public Controller getController() { | 244 | public Controller getController() { |
97 | return ctrl; | 245 | return ctrl; |
98 | } | 246 | } |
99 | 247 | ||
248 | + /** | ||
249 | + * Gets connected peers. | ||
250 | + * | ||
251 | + * @return connectedPeers from connected Peers Map. | ||
252 | + */ | ||
253 | + public ConcurrentHashMap<BGPId, BGPPeer> getConnectedPeers() { | ||
254 | + return connectedPeers; | ||
255 | + } | ||
256 | + | ||
257 | + /** | ||
258 | + * Gets peer manager. | ||
259 | + * | ||
260 | + * @return peerManager. | ||
261 | + */ | ||
262 | + public BGPPeerManager getPeerManager() { | ||
263 | + return peerManager; | ||
264 | + } | ||
265 | + | ||
100 | @Override | 266 | @Override |
101 | public BGPCfg getConfig() { | 267 | public BGPCfg getConfig() { |
102 | return this.bgpconfig; | 268 | return this.bgpconfig; |
103 | } | 269 | } |
270 | + | ||
271 | + @Override | ||
272 | + public int getBGPConnNumber() { | ||
273 | + return connectedPeers.size(); | ||
274 | + } | ||
104 | } | 275 | } |
... | \ No newline at end of file | ... | \ No newline at end of file | ... | ... |
1 | +/* | ||
2 | + * Copyright 2015 Open Networking Laboratory | ||
3 | + * | ||
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | + * you may not use this file except in compliance with the License. | ||
6 | + * You may obtain a copy of the License at | ||
7 | + * | ||
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | + * | ||
10 | + * Unless required by applicable law or agreed to in writing, software | ||
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | + * See the License for the specific language governing permissions and | ||
14 | + * limitations under the License. | ||
15 | + */ | ||
16 | + | ||
17 | +package org.onosproject.bgp.controller.impl; | ||
18 | + | ||
19 | +import java.util.Timer; | ||
20 | +import java.util.TimerTask; | ||
21 | + | ||
22 | +import org.slf4j.Logger; | ||
23 | +import org.slf4j.LoggerFactory; | ||
24 | + | ||
25 | +/** | ||
26 | + * Implement sending keepalive message to connected peer periodically based on negotiated holdtime. | ||
27 | + */ | ||
28 | +public class BGPKeepAliveTimer { | ||
29 | + | ||
30 | + private Timer keepAliveTimer; | ||
31 | + private BGPChannelHandler handler; | ||
32 | + private static final Logger log = LoggerFactory.getLogger(BGPKeepAliveTimer.class); | ||
33 | + | ||
34 | + /** | ||
35 | + * Gets keepalive timer object. | ||
36 | + * | ||
37 | + * @return keepAliveTimer keepalive timer. | ||
38 | + */ | ||
39 | + public Timer getKeepAliveTimer() { | ||
40 | + return keepAliveTimer; | ||
41 | + } | ||
42 | + | ||
43 | + /** | ||
44 | + * Initialize timer to send keepalive message periodically. | ||
45 | + * | ||
46 | + * @param h channel handler | ||
47 | + * @param seconds time interval. | ||
48 | + */ | ||
49 | + public BGPKeepAliveTimer(BGPChannelHandler h, int seconds) { | ||
50 | + this.handler = h; | ||
51 | + this.keepAliveTimer = new Timer(); | ||
52 | + this.keepAliveTimer.schedule(new SendKeepAlive(), 0, seconds * 1000); | ||
53 | + } | ||
54 | + | ||
55 | + /** | ||
56 | + * Send keepalive message to connected peer on schedule. | ||
57 | + */ | ||
58 | + class SendKeepAlive extends TimerTask { | ||
59 | + @Override | ||
60 | + public void run() { | ||
61 | + log.debug("Sending periodic KeepAlive"); | ||
62 | + | ||
63 | + try { | ||
64 | + // Send keep alive message | ||
65 | + handler.sendKeepAliveMessage(); | ||
66 | + handler.getBgpPacketStats().addOutPacket(); | ||
67 | + } catch (Exception e) { | ||
68 | + log.info("Exception occured while sending keepAlive message" + e.toString()); | ||
69 | + } | ||
70 | + } | ||
71 | + } | ||
72 | +} |
... | @@ -43,8 +43,7 @@ public class BGPPacketStatsImpl implements BGPPacketStats { | ... | @@ -43,8 +43,7 @@ public class BGPPacketStatsImpl implements BGPPacketStats { |
43 | /** | 43 | /** |
44 | * Get the outgoing packet count number. | 44 | * Get the outgoing packet count number. |
45 | * | 45 | * |
46 | - * @return | 46 | + * @return packet count |
47 | - * packet count | ||
48 | */ | 47 | */ |
49 | public int outPacketCount() { | 48 | public int outPacketCount() { |
50 | return outPacketCount; | 49 | return outPacketCount; |
... | @@ -53,8 +52,7 @@ public class BGPPacketStatsImpl implements BGPPacketStats { | ... | @@ -53,8 +52,7 @@ public class BGPPacketStatsImpl implements BGPPacketStats { |
53 | /** | 52 | /** |
54 | * Get the incoming packet count number. | 53 | * Get the incoming packet count number. |
55 | * | 54 | * |
56 | - * @return | 55 | + * @return packet count |
57 | - * packet count | ||
58 | */ | 56 | */ |
59 | public int inPacketCount() { | 57 | public int inPacketCount() { |
60 | return inPacketCount; | 58 | return inPacketCount; |
... | @@ -63,8 +61,7 @@ public class BGPPacketStatsImpl implements BGPPacketStats { | ... | @@ -63,8 +61,7 @@ public class BGPPacketStatsImpl implements BGPPacketStats { |
63 | /** | 61 | /** |
64 | * Get the wrong packet count number. | 62 | * Get the wrong packet count number. |
65 | * | 63 | * |
66 | - * @return | 64 | + * @return packet count |
67 | - * packet count | ||
68 | */ | 65 | */ |
69 | public int wrongPacketCount() { | 66 | public int wrongPacketCount() { |
70 | return wrongPacketCount; | 67 | return wrongPacketCount; |
... | @@ -110,8 +107,7 @@ public class BGPPacketStatsImpl implements BGPPacketStats { | ... | @@ -110,8 +107,7 @@ public class BGPPacketStatsImpl implements BGPPacketStats { |
110 | /** | 107 | /** |
111 | * Get the time. | 108 | * Get the time. |
112 | * | 109 | * |
113 | - * @return | 110 | + * @return time |
114 | - * time | ||
115 | */ | 111 | */ |
116 | public long getTime() { | 112 | public long getTime() { |
117 | return this.time; | 113 | return this.time; | ... | ... |
1 | +/* | ||
2 | + * Copyright 2015 Open Networking Laboratory | ||
3 | + * | ||
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | + * you may not use this file except in compliance with the License. | ||
6 | + * You may obtain a copy of the License at | ||
7 | + * | ||
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | + * | ||
10 | + * Unless required by applicable law or agreed to in writing, software | ||
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | + * See the License for the specific language governing permissions and | ||
14 | + * limitations under the License. | ||
15 | + */ | ||
16 | + | ||
17 | +package org.onosproject.bgp.controller.impl; | ||
18 | + | ||
19 | +import java.net.InetSocketAddress; | ||
20 | +import java.net.SocketAddress; | ||
21 | +import java.util.Collections; | ||
22 | +import java.util.List; | ||
23 | +import java.util.concurrent.RejectedExecutionException; | ||
24 | + | ||
25 | +import org.jboss.netty.channel.Channel; | ||
26 | +import org.onlab.packet.IpAddress; | ||
27 | +import org.onosproject.bgp.controller.BGPId; | ||
28 | +import org.onosproject.bgp.controller.BGPPacketStats; | ||
29 | +import org.onosproject.bgp.controller.BGPPeer; | ||
30 | +import org.onosproject.bgpio.protocol.BGPMessage; | ||
31 | +import org.onosproject.bgpio.protocol.BGPVersion; | ||
32 | +import org.slf4j.Logger; | ||
33 | +import org.slf4j.LoggerFactory; | ||
34 | + | ||
35 | +import com.google.common.base.MoreObjects; | ||
36 | + | ||
37 | +/** | ||
38 | + * BGPPeerImpl implements BGPPeer, maintains peer information and store updates in RIB . | ||
39 | + */ | ||
40 | +public class BGPPeerImpl implements BGPPeer { | ||
41 | + | ||
42 | + protected final Logger log = LoggerFactory.getLogger(BGPPeerImpl.class); | ||
43 | + | ||
44 | + private static final String SHUTDOWN_MSG = "Worker has already been shutdown"; | ||
45 | + | ||
46 | + private Channel channel; | ||
47 | + protected String channelId; | ||
48 | + private boolean connected; | ||
49 | + protected boolean isHandShakeComplete = false; | ||
50 | + public BGPSessionInfo sessionInfo; | ||
51 | + private BGPPacketStatsImpl pktStats; | ||
52 | + | ||
53 | + @Override | ||
54 | + public void init(BGPId bgpId, BGPVersion bgpVersion, BGPPacketStats pktStats) { | ||
55 | + this.sessionInfo.setRemoteBgpId(bgpId); | ||
56 | + this.sessionInfo.setRemoteBgpVersion(bgpVersion); | ||
57 | + this.pktStats = (BGPPacketStatsImpl) pktStats; | ||
58 | + this.sessionInfo = new BGPSessionInfo(); | ||
59 | + } | ||
60 | + | ||
61 | + // ************************ | ||
62 | + // Channel related | ||
63 | + // ************************ | ||
64 | + | ||
65 | + @Override | ||
66 | + public final void disconnectPeer() { | ||
67 | + this.channel.close(); | ||
68 | + } | ||
69 | + | ||
70 | + @Override | ||
71 | + public final void sendMessage(BGPMessage m) { | ||
72 | + log.debug("Sending message to {}", channel.getRemoteAddress()); | ||
73 | + try { | ||
74 | + channel.write(Collections.singletonList(m)); | ||
75 | + this.pktStats.addOutPacket(); | ||
76 | + } catch (RejectedExecutionException e) { | ||
77 | + log.warn(e.getMessage()); | ||
78 | + if (!e.getMessage().contains(SHUTDOWN_MSG)) { | ||
79 | + throw e; | ||
80 | + } | ||
81 | + } | ||
82 | + } | ||
83 | + | ||
84 | + @Override | ||
85 | + public final void sendMessage(List<BGPMessage> msgs) { | ||
86 | + try { | ||
87 | + channel.write(msgs); | ||
88 | + this.pktStats.addOutPacket(msgs.size()); | ||
89 | + } catch (RejectedExecutionException e) { | ||
90 | + log.warn(e.getMessage()); | ||
91 | + if (!e.getMessage().contains(SHUTDOWN_MSG)) { | ||
92 | + throw e; | ||
93 | + } | ||
94 | + } | ||
95 | + } | ||
96 | + | ||
97 | + @Override | ||
98 | + public final boolean isConnected() { | ||
99 | + return this.connected; | ||
100 | + } | ||
101 | + | ||
102 | + @Override | ||
103 | + public final void setConnected(boolean connected) { | ||
104 | + this.connected = connected; | ||
105 | + }; | ||
106 | + | ||
107 | + @Override | ||
108 | + public final void setChannel(Channel channel) { | ||
109 | + this.channel = channel; | ||
110 | + final SocketAddress address = channel.getRemoteAddress(); | ||
111 | + if (address instanceof InetSocketAddress) { | ||
112 | + final InetSocketAddress inetAddress = (InetSocketAddress) address; | ||
113 | + final IpAddress ipAddress = IpAddress.valueOf(inetAddress.getAddress()); | ||
114 | + if (ipAddress.isIp4()) { | ||
115 | + channelId = ipAddress.toString() + ':' + inetAddress.getPort(); | ||
116 | + } else { | ||
117 | + channelId = '[' + ipAddress.toString() + "]:" + inetAddress.getPort(); | ||
118 | + } | ||
119 | + } | ||
120 | + }; | ||
121 | + | ||
122 | + @Override | ||
123 | + public final Channel getChannel() { | ||
124 | + return this.channel; | ||
125 | + }; | ||
126 | + | ||
127 | + @Override | ||
128 | + public String channelId() { | ||
129 | + return channelId; | ||
130 | + } | ||
131 | + | ||
132 | + // ************************ | ||
133 | + // BGP Peer features related | ||
134 | + // ************************ | ||
135 | + | ||
136 | + @Override | ||
137 | + public final BGPId getBGPId() { | ||
138 | + return this.sessionInfo.getRemoteBgpId(); | ||
139 | + }; | ||
140 | + | ||
141 | + @Override | ||
142 | + public final String getStringId() { | ||
143 | + return this.sessionInfo.getRemoteBgpId().toString(); | ||
144 | + } | ||
145 | + | ||
146 | + @Override | ||
147 | + public final void setBgpPeerVersion(BGPVersion peerVersion) { | ||
148 | + this.sessionInfo.setRemoteBgpVersion(peerVersion); | ||
149 | + } | ||
150 | + | ||
151 | + @Override | ||
152 | + public void setBgpPeerASNum(short peerASNum) { | ||
153 | + this.sessionInfo.setRemoteBgpASNum(peerASNum); | ||
154 | + } | ||
155 | + | ||
156 | + @Override | ||
157 | + public void setBgpPeerHoldTime(short peerHoldTime) { | ||
158 | + this.sessionInfo.setRemoteBgpHoldTime(peerHoldTime); | ||
159 | + } | ||
160 | + | ||
161 | + @Override | ||
162 | + public void setBgpPeerIdentifier(int peerIdentifier) { | ||
163 | + this.sessionInfo.setRemoteBgpIdentifier(peerIdentifier); | ||
164 | + } | ||
165 | + | ||
166 | + @Override | ||
167 | + public int getBgpPeerIdentifier() { | ||
168 | + return this.sessionInfo.getRemoteBgpIdentifier(); | ||
169 | + } | ||
170 | + | ||
171 | + @Override | ||
172 | + public int getNegotiatedHoldTime() { | ||
173 | + return this.sessionInfo.getNegotiatedholdTime(); | ||
174 | + } | ||
175 | + | ||
176 | + @Override | ||
177 | + public void setNegotiatedHoldTime(short negotiatedHoldTime) { | ||
178 | + this.sessionInfo.setNegotiatedholdTime(negotiatedHoldTime); | ||
179 | + } | ||
180 | + | ||
181 | + @Override | ||
182 | + public boolean isHandshakeComplete() { | ||
183 | + return isHandShakeComplete; | ||
184 | + } | ||
185 | + | ||
186 | + @Override | ||
187 | + public String toString() { | ||
188 | + return MoreObjects.toStringHelper(getClass()).omitNullValues().add("channel", channelId()) | ||
189 | + .add("bgpId", getBGPId()).toString(); | ||
190 | + } | ||
191 | +} |
1 | +/* | ||
2 | + * Copyright 2015 Open Networking Laboratory | ||
3 | + * | ||
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | + * you may not use this file except in compliance with the License. | ||
6 | + * You may obtain a copy of the License at | ||
7 | + * | ||
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | + * | ||
10 | + * Unless required by applicable law or agreed to in writing, software | ||
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | + * See the License for the specific language governing permissions and | ||
14 | + * limitations under the License. | ||
15 | + */ | ||
16 | + | ||
17 | +package org.onosproject.bgp.controller.impl; | ||
18 | + | ||
19 | +import org.onosproject.bgp.controller.BGPId; | ||
20 | +import org.onosproject.bgpio.protocol.BGPVersion; | ||
21 | +import org.slf4j.Logger; | ||
22 | +import org.slf4j.LoggerFactory; | ||
23 | + | ||
24 | +/** | ||
25 | + * Class maintains BGP peer session info. | ||
26 | + */ | ||
27 | +public class BGPSessionInfo { | ||
28 | + | ||
29 | + protected final Logger log = LoggerFactory.getLogger(BGPSessionInfo.class); | ||
30 | + private BGPId remoteBgpId; | ||
31 | + private BGPVersion remoteBgpVersion; | ||
32 | + private short remoteBgpASNum; | ||
33 | + private short remoteBgpholdTime; | ||
34 | + private int remoteBgpIdentifier; | ||
35 | + private short negotiatedholdTime; | ||
36 | + | ||
37 | + /** | ||
38 | + * Gets the negotiated hold time for the session. | ||
39 | + * | ||
40 | + * @return negotiated hold time. | ||
41 | + */ | ||
42 | + public short getNegotiatedholdTime() { | ||
43 | + return negotiatedholdTime; | ||
44 | + } | ||
45 | + | ||
46 | + /** | ||
47 | + * Sets the negotiated hold time for the session. | ||
48 | + * | ||
49 | + * @param negotiatedholdTime negotiated hold time. | ||
50 | + */ | ||
51 | + public void setNegotiatedholdTime(short negotiatedholdTime) { | ||
52 | + this.negotiatedholdTime = negotiatedholdTime; | ||
53 | + } | ||
54 | + | ||
55 | + /** | ||
56 | + * Gets the BGP ID of BGP peer. | ||
57 | + * | ||
58 | + * @return bgp ID. | ||
59 | + */ | ||
60 | + public BGPId getRemoteBgpId() { | ||
61 | + return remoteBgpId; | ||
62 | + } | ||
63 | + | ||
64 | + /** | ||
65 | + * Sets the BGP ID of bgp peer. | ||
66 | + * | ||
67 | + * @param bgpId BGP ID to set. | ||
68 | + */ | ||
69 | + public void setRemoteBgpId(BGPId bgpId) { | ||
70 | + log.debug("Remote BGP ID {}", bgpId); | ||
71 | + this.remoteBgpId = bgpId; | ||
72 | + } | ||
73 | + | ||
74 | + /** | ||
75 | + * Gets the BGP version of peer. | ||
76 | + * | ||
77 | + * @return bgp version. | ||
78 | + */ | ||
79 | + public BGPVersion getRemoteBgpVersion() { | ||
80 | + return remoteBgpVersion; | ||
81 | + } | ||
82 | + | ||
83 | + /** | ||
84 | + * Sets the BGP version for this bgp peer. | ||
85 | + * | ||
86 | + * @param bgpVersion bgp version to set. | ||
87 | + */ | ||
88 | + public void setRemoteBgpVersion(BGPVersion bgpVersion) { | ||
89 | + log.debug("Remote BGP version {}", bgpVersion); | ||
90 | + this.remoteBgpVersion = bgpVersion; | ||
91 | + } | ||
92 | + | ||
93 | + /** | ||
94 | + * Gets the BGP remote bgp AS number. | ||
95 | + * | ||
96 | + * @return remoteBgpASNum peer AS number. | ||
97 | + */ | ||
98 | + public short getRemoteBgpASNum() { | ||
99 | + return remoteBgpASNum; | ||
100 | + } | ||
101 | + | ||
102 | + /** | ||
103 | + * Sets the AS Number for this bgp peer. | ||
104 | + * | ||
105 | + * @param bgpASNum the autonomous system number value to set. | ||
106 | + */ | ||
107 | + public void setRemoteBgpASNum(short bgpASNum) { | ||
108 | + log.debug("Remote BGP AS number {}", bgpASNum); | ||
109 | + this.remoteBgpASNum = bgpASNum; | ||
110 | + } | ||
111 | + | ||
112 | + /** | ||
113 | + * Gets the BGP peer hold time. | ||
114 | + * | ||
115 | + * @return bgp hold time. | ||
116 | + */ | ||
117 | + public short getRemoteBgpHoldTime() { | ||
118 | + return remoteBgpholdTime; | ||
119 | + } | ||
120 | + | ||
121 | + /** | ||
122 | + * Sets the hold time for this bgp peer. | ||
123 | + * | ||
124 | + * @param holdTime the hold timer value to set. | ||
125 | + */ | ||
126 | + public void setRemoteBgpHoldTime(short holdTime) { | ||
127 | + log.debug("Remote BGP HoldTime {}", holdTime); | ||
128 | + this.remoteBgpholdTime = holdTime; | ||
129 | + } | ||
130 | + | ||
131 | + /** | ||
132 | + * Gets the BGP version for this bgp peer. | ||
133 | + * | ||
134 | + * @return bgp identifier. | ||
135 | + */ | ||
136 | + public int getRemoteBgpIdentifier() { | ||
137 | + return remoteBgpIdentifier; | ||
138 | + } | ||
139 | + | ||
140 | + /** | ||
141 | + * Sets the peer identifier value. | ||
142 | + * | ||
143 | + * @param bgpIdentifier the bgp peer identifier value. | ||
144 | + */ | ||
145 | + public void setRemoteBgpIdentifier(int bgpIdentifier) { | ||
146 | + log.debug("Remote BGP Identifier {}", bgpIdentifier); | ||
147 | + this.remoteBgpIdentifier = bgpIdentifier; | ||
148 | + } | ||
149 | +} |
-
Please register or login to post a comment