홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
tom
2d6d397b 1 parent 2868040f

More IO loop work.

Expand all
Inline Side-by-side
Showing 6 changed files with 126 additions and 83 deletions
1 package org.onlab.nio; 1 package org.onlab.nio;
2 2
3 +import org.onlab.util.Counter;
3 import org.slf4j.Logger; 4 import org.slf4j.Logger;
4 import org.slf4j.LoggerFactory; 5 import org.slf4j.LoggerFactory;
5 6
...@@ -40,6 +41,10 @@ public abstract class MessageStream<M extends Message> { ...@@ -40,6 +41,10 @@ public abstract class MessageStream<M extends Message> {
40 private Exception ioError; 41 private Exception ioError;
41 private long lastActiveTime; 42 private long lastActiveTime;
42 43
44 + private final Counter bytesIn = new Counter();
45 + private final Counter messagesIn = new Counter();
46 + private final Counter bytesOut = new Counter();
47 + private final Counter messagesOut = new Counter();
43 48
44 /** 49 /**
45 * Creates a message stream associated with the specified IO loop and 50 * Creates a message stream associated with the specified IO loop and
...@@ -93,6 +98,11 @@ public abstract class MessageStream<M extends Message> { ...@@ -93,6 +98,11 @@ public abstract class MessageStream<M extends Message> {
93 closed = true; 98 closed = true;
94 } 99 }
95 100
101 + bytesIn.freeze();
102 + bytesOut.freeze();
103 + messagesIn.freeze();
104 + messagesOut.freeze();
105 +
96 loop.removeStream(this); 106 loop.removeStream(this);
97 if (key != null) { 107 if (key != null) {
98 try { 108 try {
...@@ -176,6 +186,8 @@ public abstract class MessageStream<M extends Message> { ...@@ -176,6 +186,8 @@ public abstract class MessageStream<M extends Message> {
176 inbound.flip(); 186 inbound.flip();
177 while ((message = read(inbound)) != null) { 187 while ((message = read(inbound)) != null) {
178 messages.add(message); 188 messages.add(message);
189 + messagesIn.add(1);
190 + bytesIn.add(message.length());
179 } 191 }
180 inbound.compact(); 192 inbound.compact();
181 193
...@@ -226,8 +238,9 @@ public abstract class MessageStream<M extends Message> { ...@@ -226,8 +238,9 @@ public abstract class MessageStream<M extends Message> {
226 while (outbound.remaining() < message.length()) { 238 while (outbound.remaining() < message.length()) {
227 doubleSize(); 239 doubleSize();
228 } 240 }
229 - // Place the message into the buffer and bump the output trackers.
230 write(message, outbound); 241 write(message, outbound);
242 + messagesOut.add(1);
243 + bytesOut.add(message.length());
231 } 244 }
232 245
233 // Forces a flush, unless one is planned already. 246 // Forces a flush, unless one is planned already.
...@@ -273,6 +286,18 @@ public abstract class MessageStream<M extends Message> { ...@@ -273,6 +286,18 @@ public abstract class MessageStream<M extends Message> {
273 } 286 }
274 } 287 }
275 288
289 +
290 + /**
291 + * Indicates whether data has been written but not flushed yet.
292 + *
293 + * @return true if flush is required
294 + */
295 + boolean isFlushRequired() {
296 + synchronized (this) {
297 + return outbound.position() > 0;
298 + }
299 + }
300 +
276 /** 301 /**
277 * Attempts to flush data, internal stream state and channel availability 302 * Attempts to flush data, internal stream state and channel availability
278 * permitting. Invoked by the driver I/O loop during handling of writable 303 * permitting. Invoked by the driver I/O loop during handling of writable
...@@ -344,4 +369,40 @@ public abstract class MessageStream<M extends Message> { ...@@ -344,4 +369,40 @@ public abstract class MessageStream<M extends Message> {
344 return currentTimeMillis() - lastActiveTime > maxIdleMillis() && key != null; 369 return currentTimeMillis() - lastActiveTime > maxIdleMillis() && key != null;
345 } 370 }
346 371
372 + /**
373 + * Returns the inbound bytes counter.
374 + *
375 + * @return inbound bytes counter
376 + */
377 + public Counter bytesIn() {
378 + return bytesIn;
379 + }
380 +
381 + /**
382 + * Returns the outbound bytes counter.
383 + *
384 + * @return outbound bytes counter
385 + */
386 + public Counter bytesOut() {
387 + return bytesOut;
388 + }
389 +
390 + /**
391 + * Returns the inbound messages counter.
392 + *
393 + * @return inbound messages counter
394 + */
395 + public Counter messagesIn() {
396 + return messagesIn;
397 + }
398 +
399 + /**
400 + * Returns the outbound messages counter.
401 + *
402 + * @return outbound messages counter
403 + */
404 + public Counter messagesOut() {
405 + return messagesOut;
406 + }
407 +
347 } 408 }
......
1 +/**
2 + * Mechanism to transfer messages over network using IO loop and
3 + * message stream, backed by NIO byte buffers.
4 + */
5 +package org.onlab.nio;
...\ No newline at end of file ...\ No newline at end of file
1 package org.onlab.nio; 1 package org.onlab.nio;
2 2
3 +import org.onlab.util.Counter;
3 import org.slf4j.Logger; 4 import org.slf4j.Logger;
4 import org.slf4j.LoggerFactory; 5 import org.slf4j.LoggerFactory;
5 6
...@@ -10,6 +11,7 @@ import java.net.SocketAddress; ...@@ -10,6 +11,7 @@ import java.net.SocketAddress;
10 import java.nio.channels.ByteChannel; 11 import java.nio.channels.ByteChannel;
11 import java.nio.channels.SelectionKey; 12 import java.nio.channels.SelectionKey;
12 import java.nio.channels.SocketChannel; 13 import java.nio.channels.SocketChannel;
14 +import java.text.DecimalFormat;
13 import java.util.ArrayList; 15 import java.util.ArrayList;
14 import java.util.List; 16 import java.util.List;
15 import java.util.concurrent.ExecutionException; 17 import java.util.concurrent.ExecutionException;
...@@ -26,9 +28,9 @@ import static org.onlab.util.Tools.namedThreads; ...@@ -26,9 +28,9 @@ import static org.onlab.util.Tools.namedThreads;
26 /** 28 /**
27 * Auxiliary test fixture to measure speed of NIO-based channels. 29 * Auxiliary test fixture to measure speed of NIO-based channels.
28 */ 30 */
29 -public class StandaloneSpeedClient { 31 +public class IOLoopClient {
30 32
31 - private static Logger log = LoggerFactory.getLogger(StandaloneSpeedClient.class); 33 + private static Logger log = LoggerFactory.getLogger(IOLoopClient.class);
32 34
33 private final InetAddress ip; 35 private final InetAddress ip;
34 private final int port; 36 private final int port;
...@@ -39,8 +41,8 @@ public class StandaloneSpeedClient { ...@@ -39,8 +41,8 @@ public class StandaloneSpeedClient {
39 private final ExecutorService ipool; 41 private final ExecutorService ipool;
40 private final ExecutorService wpool; 42 private final ExecutorService wpool;
41 43
42 -// ThroughputTracker messages; 44 + Counter messages;
43 -// ThroughputTracker bytes; 45 + Counter bytes;
44 46
45 /** 47 /**
46 * Main entry point to launch the client. 48 * Main entry point to launch the client.
...@@ -61,7 +63,7 @@ public class StandaloneSpeedClient { ...@@ -61,7 +63,7 @@ public class StandaloneSpeedClient {
61 63
62 log.info("Setting up client with {} workers sending {} {}-byte messages to {} server... ", 64 log.info("Setting up client with {} workers sending {} {}-byte messages to {} server... ",
63 wc, mc, ml, ip); 65 wc, mc, ml, ip);
64 - StandaloneSpeedClient sc = new StandaloneSpeedClient(ip, wc, mc, ml, StandaloneSpeedServer.PORT); 66 + IOLoopClient sc = new IOLoopClient(ip, wc, mc, ml, IOLoopServer.PORT);
65 67
66 sc.start(); 68 sc.start();
67 delay(2000); 69 delay(2000);
...@@ -82,7 +84,7 @@ public class StandaloneSpeedClient { ...@@ -82,7 +84,7 @@ public class StandaloneSpeedClient {
82 * @param port socket port 84 * @param port socket port
83 * @throws IOException if unable to create IO loops 85 * @throws IOException if unable to create IO loops
84 */ 86 */
85 - public StandaloneSpeedClient(InetAddress ip, int wc, int mc, int ml, int port) throws IOException { 87 + public IOLoopClient(InetAddress ip, int wc, int mc, int ml, int port) throws IOException {
86 this.ip = ip; 88 this.ip = ip;
87 this.port = port; 89 this.port = port;
88 this.msgCount = mc; 90 this.msgCount = mc;
...@@ -101,15 +103,15 @@ public class StandaloneSpeedClient { ...@@ -101,15 +103,15 @@ public class StandaloneSpeedClient {
101 * @throws IOException if unable to open connection 103 * @throws IOException if unable to open connection
102 */ 104 */
103 public void start() throws IOException { 105 public void start() throws IOException {
104 -// messages = new ThroughputTracker(); 106 + messages = new Counter();
105 -// bytes = new ThroughputTracker(); 107 + bytes = new Counter();
106 108
107 // First start up all the IO loops 109 // First start up all the IO loops
108 for (CustomIOLoop l : iloops) { 110 for (CustomIOLoop l : iloops) {
109 ipool.execute(l); 111 ipool.execute(l);
110 } 112 }
111 113
112 -// // Wait for all of them to get going 114 + // Wait for all of them to get going
113 // for (CustomIOLoop l : iloops) 115 // for (CustomIOLoop l : iloops)
114 // l.waitForStart(TIMEOUT); 116 // l.waitForStart(TIMEOUT);
115 117
...@@ -151,20 +153,20 @@ public class StandaloneSpeedClient { ...@@ -151,20 +153,20 @@ public class StandaloneSpeedClient {
151 l.worker.task.get(secs, TimeUnit.SECONDS); 153 l.worker.task.get(secs, TimeUnit.SECONDS);
152 } 154 }
153 } 155 }
154 -// messages.freeze(); 156 + messages.freeze();
155 -// bytes.freeze(); 157 + bytes.freeze();
156 } 158 }
157 159
158 /** 160 /**
159 * Reports on the accumulated throughput trackers. 161 * Reports on the accumulated throughput trackers.
160 */ 162 */
161 public void report() { 163 public void report() {
162 -// DecimalFormat f = new DecimalFormat("#,##0"); 164 + DecimalFormat f = new DecimalFormat("#,##0");
163 -// log.info("{} messages; {} bytes; {} mps; {} Mbs", 165 + log.info("{} messages; {} bytes; {} mps; {} Mbs",
164 -// f.format(messages.total()), 166 + f.format(messages.total()),
165 -// f.format(bytes.total()), 167 + f.format(bytes.total()),
166 -// f.format(messages.throughput()), 168 + f.format(messages.throughput()),
167 -// f.format(bytes.throughput() / (1024 * 128))); 169 + f.format(bytes.throughput() / (1024 * 128)));
168 } 170 }
169 171
170 172
...@@ -187,16 +189,16 @@ public class StandaloneSpeedClient { ...@@ -187,16 +189,16 @@ public class StandaloneSpeedClient {
187 protected synchronized void removeStream(MessageStream<TestMessage> b) { 189 protected synchronized void removeStream(MessageStream<TestMessage> b) {
188 super.removeStream(b); 190 super.removeStream(b);
189 191
190 -// messages.add(b.inMessages().total()); 192 + messages.add(b.messagesIn().total());
191 -// bytes.add(b.inBytes().total()); 193 + bytes.add(b.bytesIn().total());
192 -// b.inMessages().reset(); 194 + b.messagesOut().reset();
193 -// b.inBytes().reset(); 195 + b.bytesOut().reset();
194 - 196 +//
195 -// log.info("Disconnected client; inbound {} mps, {} Mbps; outbound {} mps, {} Mbps", 197 + log.info("Disconnected client; inbound {} mps, {} Mbps; outbound {} mps, {} Mbps",
196 -// StandaloneSpeedServer.format.format(b.inMessages().throughput()), 198 + IOLoopServer.FORMAT.format(b.messagesIn().throughput()),
197 -// StandaloneSpeedServer.format.format(b.inBytes().throughput() / (1024 * 128)), 199 + IOLoopServer.FORMAT.format(b.bytesIn().throughput() / (1024 * 128)),
198 -// StandaloneSpeedServer.format.format(b.outMessages().throughput()), 200 + IOLoopServer.FORMAT.format(b.messagesOut().throughput()),
199 -// StandaloneSpeedServer.format.format(b.outBytes().throughput() / (1024 * 128))); 201 + IOLoopServer.FORMAT.format(b.bytesOut().throughput() / (1024 * 128)));
200 } 202 }
201 203
202 @Override 204 @Override
......
...@@ -48,11 +48,11 @@ public class IOLoopIntegrationTest { ...@@ -48,11 +48,11 @@ public class IOLoopIntegrationTest {
48 48
49 // Setup the test on a random port to avoid intermittent test failures 49 // Setup the test on a random port to avoid intermittent test failures
50 // due to the port being already bound. 50 // due to the port being already bound.
51 - int port = StandaloneSpeedServer.PORT + new Random().nextInt(100); 51 + int port = IOLoopServer.PORT + new Random().nextInt(100);
52 52
53 InetAddress ip = InetAddress.getLoopbackAddress(); 53 InetAddress ip = InetAddress.getLoopbackAddress();
54 - StandaloneSpeedServer sss = new StandaloneSpeedServer(ip, THREADS, size, port); 54 + IOLoopServer sss = new IOLoopServer(ip, THREADS, size, port);
55 - StandaloneSpeedClient ssc = new StandaloneSpeedClient(ip, THREADS, count, size, port); 55 + IOLoopClient ssc = new IOLoopClient(ip, THREADS, count, size, port);
56 56
57 sss.start(); 57 sss.start();
58 ssc.start(); 58 ssc.start();
...@@ -64,32 +64,6 @@ public class IOLoopIntegrationTest { ...@@ -64,32 +64,6 @@ public class IOLoopIntegrationTest {
64 delay(1000); 64 delay(1000);
65 sss.stop(); 65 sss.stop();
66 sss.report(); 66 sss.report();
67 -
68 - // Note that the client and server will have potentially significantly
69 - // differing rates. This is due to the wide variance in how tightly
70 - // the throughput tracking starts & stops relative to to the short
71 - // test duration.
72 -// System.out.println(f.format(ssc.messages.throughput()) + " mps");
73 -
74 -// // Make sure client sent everything.
75 -// assertEquals("incorrect client message count sent",
76 -// (long) count * THREADS, ssc.messages.total());
77 -// assertEquals("incorrect client bytes count sent",
78 -// (long) size * count * THREADS, ssc.bytes.total());
79 -//
80 -// // Make sure server received everything.
81 -// assertEquals("incorrect server message count received",
82 -// (long) count * THREADS, sss.messages.total());
83 -// assertEquals("incorrect server bytes count received",
84 -// (long) size * count * THREADS, sss.bytes.total());
85 -//
86 -// // Make sure speeds were reasonable.
87 -// if (mps > 0.0) {
88 -// assertAboveThreshold("insufficient client speed", mps,
89 -// ssc.messages.throughput());
90 -// assertAboveThreshold("insufficient server speed", mps / 2,
91 -// sss.messages.throughput());
92 -// }
93 } 67 }
94 68
95 } 69 }
......
1 package org.onlab.nio; 1 package org.onlab.nio;
2 2
3 +import org.onlab.util.Counter;
3 import org.slf4j.Logger; 4 import org.slf4j.Logger;
4 import org.slf4j.LoggerFactory; 5 import org.slf4j.LoggerFactory;
5 6
...@@ -23,9 +24,9 @@ import static org.onlab.util.Tools.namedThreads; ...@@ -23,9 +24,9 @@ import static org.onlab.util.Tools.namedThreads;
23 /** 24 /**
24 * Auxiliary test fixture to measure speed of NIO-based channels. 25 * Auxiliary test fixture to measure speed of NIO-based channels.
25 */ 26 */
26 -public class StandaloneSpeedServer { 27 +public class IOLoopServer {
27 28
28 - private static Logger log = LoggerFactory.getLogger(StandaloneSpeedServer.class); 29 + private static Logger log = LoggerFactory.getLogger(IOLoopServer.class);
29 30
30 private static final int PRUNE_FREQUENCY = 1000; 31 private static final int PRUNE_FREQUENCY = 1000;
31 32
...@@ -48,8 +49,8 @@ public class StandaloneSpeedServer { ...@@ -48,8 +49,8 @@ public class StandaloneSpeedServer {
48 private final int msgLength; 49 private final int msgLength;
49 private int lastWorker = -1; 50 private int lastWorker = -1;
50 51
51 -// ThroughputTracker messages; 52 + Counter messages;
52 -// ThroughputTracker bytes; 53 + Counter bytes;
53 54
54 /** 55 /**
55 * Main entry point to launch the server. 56 * Main entry point to launch the server.
...@@ -64,7 +65,7 @@ public class StandaloneSpeedServer { ...@@ -64,7 +65,7 @@ public class StandaloneSpeedServer {
64 65
65 log.info("Setting up the server with {} workers, {} byte messages on {}... ", 66 log.info("Setting up the server with {} workers, {} byte messages on {}... ",
66 wc, ml, ip); 67 wc, ml, ip);
67 - StandaloneSpeedServer ss = new StandaloneSpeedServer(ip, wc, ml, PORT); 68 + IOLoopServer ss = new IOLoopServer(ip, wc, ml, PORT);
68 ss.start(); 69 ss.start();
69 70
70 // Start pruning clients. 71 // Start pruning clients.
...@@ -83,7 +84,7 @@ public class StandaloneSpeedServer { ...@@ -83,7 +84,7 @@ public class StandaloneSpeedServer {
83 * @param port listen port 84 * @param port listen port
84 * @throws IOException if unable to create IO loops 85 * @throws IOException if unable to create IO loops
85 */ 86 */
86 - public StandaloneSpeedServer(InetAddress ip, int wc, int ml, int port) throws IOException { 87 + public IOLoopServer(InetAddress ip, int wc, int ml, int port) throws IOException {
87 this.workerCount = wc; 88 this.workerCount = wc;
88 this.msgLength = ml; 89 this.msgLength = ml;
89 this.ipool = Executors.newFixedThreadPool(workerCount, namedThreads("io-loop")); 90 this.ipool = Executors.newFixedThreadPool(workerCount, namedThreads("io-loop"));
...@@ -98,14 +99,14 @@ public class StandaloneSpeedServer { ...@@ -98,14 +99,14 @@ public class StandaloneSpeedServer {
98 * Start the server IO loops and kicks off throughput tracking. 99 * Start the server IO loops and kicks off throughput tracking.
99 */ 100 */
100 public void start() { 101 public void start() {
101 -// messages = new ThroughputTracker(); 102 + messages = new Counter();
102 -// bytes = new ThroughputTracker(); 103 + bytes = new Counter();
103 104
104 for (CustomIOLoop l : iloops) { 105 for (CustomIOLoop l : iloops) {
105 ipool.execute(l); 106 ipool.execute(l);
106 } 107 }
107 apool.execute(aloop); 108 apool.execute(aloop);
108 -// 109 +
109 // for (CustomIOLoop l : iloops) 110 // for (CustomIOLoop l : iloops)
110 // l.waitForStart(TIMEOUT); 111 // l.waitForStart(TIMEOUT);
111 // aloop.waitForStart(TIMEOUT); 112 // aloop.waitForStart(TIMEOUT);
...@@ -124,20 +125,20 @@ public class StandaloneSpeedServer { ...@@ -124,20 +125,20 @@ public class StandaloneSpeedServer {
124 // l.waitForFinish(TIMEOUT); 125 // l.waitForFinish(TIMEOUT);
125 // aloop.waitForFinish(TIMEOUT); 126 // aloop.waitForFinish(TIMEOUT);
126 // 127 //
127 -// messages.freeze(); 128 + messages.freeze();
128 -// bytes.freeze(); 129 + bytes.freeze();
129 } 130 }
130 131
131 /** 132 /**
132 * Reports on the accumulated throughput trackers. 133 * Reports on the accumulated throughput trackers.
133 */ 134 */
134 public void report() { 135 public void report() {
135 -// DecimalFormat f = new DecimalFormat("#,##0"); 136 + DecimalFormat f = new DecimalFormat("#,##0");
136 -// log.info("{} messages; {} bytes; {} mps; {} Mbs", 137 + log.info("{} messages; {} bytes; {} mps; {} Mbs",
137 -// f.format(messages.total()), 138 + f.format(messages.total()),
138 -// f.format(bytes.total()), 139 + f.format(bytes.total()),
139 -// f.format(messages.throughput()), 140 + f.format(messages.throughput()),
140 -// f.format(bytes.throughput() / (1024 * 128))); 141 + f.format(bytes.throughput() / (1024 * 128)));
141 } 142 }
142 143
143 /** 144 /**
...@@ -170,15 +171,15 @@ public class StandaloneSpeedServer { ...@@ -170,15 +171,15 @@ public class StandaloneSpeedServer {
170 @Override 171 @Override
171 protected void removeStream(MessageStream<TestMessage> stream) { 172 protected void removeStream(MessageStream<TestMessage> stream) {
172 super.removeStream(stream); 173 super.removeStream(stream);
173 -// 174 +
174 -// messages.add(b.inMessages().total()); 175 + messages.add(stream.messagesIn().total());
175 -// bytes.add(b.inBytes().total()); 176 + bytes.add(stream.bytesIn().total());
176 -// 177 +
177 -// log.info("Disconnected client; inbound {} mps, {} Mbps; outbound {} mps, {} Mbps", 178 + log.info("Disconnected client; inbound {} mps, {} Mbps; outbound {} mps, {} Mbps",
178 -// format.format(b.inMessages().throughput()), 179 + FORMAT.format(stream.messagesIn().throughput()),
179 -// format.format(b.inBytes().throughput() / (1024 * 128)), 180 + FORMAT.format(stream.bytesIn().throughput() / (1024 * 128)),
180 -// format.format(b.outMessages().throughput()), 181 + FORMAT.format(stream.messagesOut().throughput()),
181 -// format.format(b.outBytes().throughput() / (1024 * 128))); 182 + FORMAT.format(stream.bytesOut().throughput() / (1024 * 128)));
182 } 183 }
183 184
184 @Override 185 @Override
......