Added ability to measure round-trip latency and to assure message integrity.

 package org.onlab.onos.foo;
 package org.onlab.onos.foo;
 
 
+import com.google.common.collect.Lists;
 import org.onlab.nio.IOLoop;
 import org.onlab.nio.IOLoop;
 import org.onlab.nio.MessageStream;
 import org.onlab.nio.MessageStream;
 import org.onlab.util.Counter;
 import org.onlab.util.Counter;
@@ -25,6 +26,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.TimeoutException;
 
 
 import static java.lang.String.format;
 import static java.lang.String.format;
+import static java.lang.System.currentTimeMillis;
 import static java.lang.System.out;
 import static java.lang.System.out;
 import static org.onlab.onos.foo.IOLoopTestServer.PORT;
 import static org.onlab.onos.foo.IOLoopTestServer.PORT;
 import static org.onlab.util.Tools.delay;
 import static org.onlab.util.Tools.delay;
@@ -48,15 +50,18 @@ public class IOLoopTestClient {
 
 
     Counter messages;
     Counter messages;
     Counter bytes;
     Counter bytes;
+    long latencyTotal = 0;
+    long latencyCount = 0;
+
 
 
     /**
     /**
      * Main entry point to launch the client.
      * Main entry point to launch the client.
      *
      *
      * @param args command-line arguments
      * @param args command-line arguments
-     * @throws java.io.IOException          if unable to connect to server
-     * @throws InterruptedException if latch wait gets interrupted
-     * @throws java.util.concurrent.ExecutionException   if wait gets interrupted
-     * @throws java.util.concurrent.TimeoutException     if timeout occurred while waiting for completion
+     * @throws java.io.IOException                     if unable to connect to server
+     * @throws InterruptedException                    if latch wait gets interrupted
+     * @throws java.util.concurrent.ExecutionException if wait gets interrupted
+     * @throws java.util.concurrent.TimeoutException   if timeout occurred while waiting for completion
      */
      */
     public static void main(String[] args)
     public static void main(String[] args)
             throws IOException, InterruptedException, ExecutionException, TimeoutException {
             throws IOException, InterruptedException, ExecutionException, TimeoutException {
@@ -158,15 +163,17 @@ public class IOLoopTestClient {
      * Waits for the client workers to complete.
      * Waits for the client workers to complete.
      *
      *
      * @param secs timeout in seconds
      * @param secs timeout in seconds
-     * @throws java.util.concurrent.ExecutionException   if execution failed
-     * @throws InterruptedException if interrupt occurred while waiting
-     * @throws java.util.concurrent.TimeoutException     if timeout occurred
+     * @throws java.util.concurrent.ExecutionException if execution failed
+     * @throws InterruptedException                    if interrupt occurred while waiting
+     * @throws java.util.concurrent.TimeoutException   if timeout occurred
      */
      */
     public void await(int secs) throws InterruptedException,
     public void await(int secs) throws InterruptedException,
             ExecutionException, TimeoutException {
             ExecutionException, TimeoutException {
         for (CustomIOLoop l : iloops) {
         for (CustomIOLoop l : iloops) {
             if (l.worker.task != null) {
             if (l.worker.task != null) {
                 l.worker.task.get(secs, TimeUnit.SECONDS);
                 l.worker.task.get(secs, TimeUnit.SECONDS);
+                latencyTotal += l.latencyTotal;
+                latencyCount += l.latencyCount;
             }
             }
         }
         }
         messages.freeze();
         messages.freeze();
@@ -178,10 +185,11 @@ public class IOLoopTestClient {
      */
      */
     public void report() {
     public void report() {
         DecimalFormat f = new DecimalFormat("#,##0");
         DecimalFormat f = new DecimalFormat("#,##0");
-        out.println(format("Client: %s messages; %s bytes; %s mps; %s Mbs",
+        out.println(format("Client: %s messages; %s bytes; %s mps; %s Mbs; %s ms latency",
                            f.format(messages.total()), f.format(bytes.total()),
                            f.format(messages.total()), f.format(bytes.total()),
                            f.format(messages.throughput()),
                            f.format(messages.throughput()),
-                           f.format(bytes.throughput() / (1024 * msgLength))));
+                           f.format(bytes.throughput() / (1024 * msgLength)),
+                           f.format(latencyTotal / latencyCount)));
     }
     }
 
 
 
 
@@ -189,6 +197,9 @@ public class IOLoopTestClient {
     private class CustomIOLoop extends IOLoop<TestMessage, TestMessageStream> {
     private class CustomIOLoop extends IOLoop<TestMessage, TestMessageStream> {
 
 
         Worker worker = new Worker();
         Worker worker = new Worker();
+        long latencyTotal = 0;
+        long latencyCount = 0;
+
 
 
         public CustomIOLoop() throws IOException {
         public CustomIOLoop() throws IOException {
             super(500);
             super(500);
@@ -219,7 +230,12 @@ public class IOLoopTestClient {
 
 
         @Override
         @Override
         protected void processMessages(List<TestMessage> messages,
         protected void processMessages(List<TestMessage> messages,
-                                       MessageStream<TestMessage> b) {
+                                       MessageStream<TestMessage> stream) {
+            for (TestMessage message : messages) {
+                // TODO: summarize latency data better
+                latencyTotal += currentTimeMillis() - message.requestorTime();
+                latencyCount++;
+            }
             worker.release(messages.size());
             worker.release(messages.size());
         }
         }
 
 
@@ -241,15 +257,15 @@ public class IOLoopTestClient {
         private static final int BATCH_SIZE = 1000;
         private static final int BATCH_SIZE = 1000;
         private static final int PERMITS = 2 * BATCH_SIZE;
         private static final int PERMITS = 2 * BATCH_SIZE;
 
 
-        private TestMessageStream b;
+        private TestMessageStream stream;
         private FutureTask<Worker> task;
         private FutureTask<Worker> task;
 
 
         // Stuff to throttle pump
         // Stuff to throttle pump
         private final Semaphore semaphore = new Semaphore(PERMITS);
         private final Semaphore semaphore = new Semaphore(PERMITS);
         private int msgWritten;
         private int msgWritten;
 
 
-        void pump(TestMessageStream b) {
-            this.b = b;
+        void pump(TestMessageStream stream) {
+            this.stream = stream;
             task = new FutureTask<>(this, this);
             task = new FutureTask<>(this, this);
             wpool.execute(task);
             wpool.execute(task);
         }
         }
@@ -259,18 +275,15 @@ public class IOLoopTestClient {
             try {
             try {
                 log.info("Worker started...");
                 log.info("Worker started...");
 
 
-                List<TestMessage> batch = new ArrayList<>();
-                for (int i = 0; i < BATCH_SIZE; i++) {
-                    batch.add(new TestMessage(msgLength));
-                }
-
                 while (msgWritten < msgCount) {
                 while (msgWritten < msgCount) {
-                    msgWritten += writeBatch(b, batch);
+                    int size = Math.min(BATCH_SIZE, msgCount - msgWritten);
+                    writeBatch(size);
+                    msgWritten += size;
                 }
                 }
 
 
                 // Now try to get all the permits back before sending poison pill
                 // Now try to get all the permits back before sending poison pill
                 semaphore.acquireUninterruptibly(PERMITS);
                 semaphore.acquireUninterruptibly(PERMITS);
-                b.close();
+                stream.close();
 
 
                 log.info("Worker done...");
                 log.info("Worker done...");
 
 
@@ -280,18 +293,15 @@ public class IOLoopTestClient {
         }
         }
 
 
 
 
-        private int writeBatch(TestMessageStream b, List<TestMessage> batch)
-                throws IOException {
-            int count = Math.min(BATCH_SIZE, msgCount - msgWritten);
-            acquire(count);
-            if (count == BATCH_SIZE) {
-                b.write(batch);
-            } else {
-                for (int i = 0; i < count; i++) {
-                    b.write(batch.get(i));
-                }
+        private void writeBatch(int size) throws IOException {
+            // Build a batch of messages
+            List<TestMessage> batch = Lists.newArrayListWithCapacity(size);
+            for (int i = 0; i < size; i++) {
+                batch.add(new TestMessage(msgLength, currentTimeMillis(), 0,
+                                          this.stream.padding(msgLength)));
             }
             }
-            return count;
+            acquire(size);
+            stream.write(batch);
         }
         }
 
 
 
 

 package org.onlab.onos.foo;
 package org.onlab.onos.foo;
 
 
+import com.google.common.collect.Lists;
 import org.onlab.nio.AcceptorLoop;
 import org.onlab.nio.AcceptorLoop;
 import org.onlab.nio.IOLoop;
 import org.onlab.nio.IOLoop;
 import org.onlab.nio.MessageStream;
 import org.onlab.nio.MessageStream;
@@ -22,6 +23,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Executors;
 
 
 import static java.lang.String.format;
 import static java.lang.String.format;
+import static java.lang.System.currentTimeMillis;
 import static java.lang.System.out;
 import static java.lang.System.out;
 import static org.onlab.util.Tools.delay;
 import static org.onlab.util.Tools.delay;
 import static org.onlab.util.Tools.namedThreads;
 import static org.onlab.util.Tools.namedThreads;
@@ -202,11 +204,20 @@ public class IOLoopTestServer {
         protected void processMessages(List<TestMessage> messages,
         protected void processMessages(List<TestMessage> messages,
                                        MessageStream<TestMessage> stream) {
                                        MessageStream<TestMessage> stream) {
             try {
             try {
-                stream.write(messages);
+                stream.write(createResponses(messages));
             } catch (IOException e) {
             } catch (IOException e) {
                 log.error("Unable to echo messages", e);
                 log.error("Unable to echo messages", e);
             }
             }
         }
         }
+
+        private List<TestMessage> createResponses(List<TestMessage> messages) {
+            List<TestMessage> responses = Lists.newArrayListWithCapacity(messages.size());
+            for (TestMessage message : messages) {
+                responses.add(new TestMessage(message.length(), message.requestorTime(),
+                                              currentTimeMillis(), message.padding()));
+            }
+            return responses;
+        }
     }
     }
 
 
     // Loop for accepting client connections
     // Loop for accepting client connections

@@ -2,40 +2,42 @@ package org.onlab.onos.foo;
 
 
 import org.onlab.nio.AbstractMessage;
 import org.onlab.nio.AbstractMessage;
 
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 /**
 /**
- * Fixed-length message.
+ * Test message for measuring rate and round-trip latency.
  */
  */
 public class TestMessage extends AbstractMessage {
 public class TestMessage extends AbstractMessage {
 
 
-    private final byte[] data;
+    private final byte[] padding;
+
+    private final long requestorTime;
+    private final long responderTime;
 
 
     /**
     /**
-     * Creates a new message with the specified length.
+     * Creates a new message with the specified data.
      *
      *
-     * @param length message length
+     * @param requestorTime requester time
+     * @param responderTime responder time
+     * @param padding       message padding
      */
      */
-    public TestMessage(int length) {
+    TestMessage(int length, long requestorTime, long responderTime, byte[] padding) {
         this.length = length;
         this.length = length;
-        data = new byte[length];
+        this.requestorTime = requestorTime;
+        this.responderTime = responderTime;
+        this.padding = checkNotNull(padding, "Padding cannot be null");
     }
     }
 
 
-    /**
-     * Creates a new message with the specified data.
-     *
-     * @param data message data
-     */
-    TestMessage(byte[] data) {
-        this.length = data.length;
-        this.data = data;
+    public long requestorTime() {
+        return requestorTime;
     }
     }
 
 
-    /**
-     * Gets the backing byte array data.
-     *
-     * @return backing byte array
-     */
-    public byte[] data() {
-        return data;
+    public long responderTime() {
+        return responderTime;
+    }
+
+    public byte[] padding() {
+        return padding;
     }
     }
 
 
 }
 }

@@ -6,24 +6,21 @@ import org.onlab.nio.MessageStream;
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
 import java.nio.channels.ByteChannel;
 import java.nio.channels.ByteChannel;
 
 
+import static com.google.common.base.Preconditions.checkState;
+
 /**
 /**
  * Fixed-length message transfer buffer.
  * Fixed-length message transfer buffer.
  */
  */
 public class TestMessageStream extends MessageStream<TestMessage> {
 public class TestMessageStream extends MessageStream<TestMessage> {
 
 
     private static final String E_WRONG_LEN = "Illegal message length: ";
     private static final String E_WRONG_LEN = "Illegal message length: ";
+    private static final long START_TAG = 0xfeedcafedeaddeedL;
+    private static final long END_TAG = 0xbeadcafedeaddeedL;
+    private static final int META_LENGTH = 40;
 
 
     private final int length;
     private final int length;
 
 
-    /**
-     * Create a new buffer for transferring messages of the specified length.
-     *
-     * @param length message length
-     * @param ch     backing channel
-     * @param loop   driver loop
-     */
-    public TestMessageStream(int length, ByteChannel ch,
-                             IOLoop<TestMessage, ?> loop) {
+    public TestMessageStream(int length, ByteChannel ch, IOLoop<TestMessage, ?> loop) {
         super(loop, ch, 64 * 1024, 500);
         super(loop, ch, 64 * 1024, 500);
         this.length = length;
         this.length = length;
     }
     }
@@ -33,26 +30,37 @@ public class TestMessageStream extends MessageStream<TestMessage> {
         if (rb.remaining() < length) {
         if (rb.remaining() < length) {
             return null;
             return null;
         }
         }
-        TestMessage message = new TestMessage(length);
-        rb.get(message.data());
-        return message;
+
+        long startTag = rb.getLong();
+        checkState(startTag == START_TAG, "Incorrect message start");
+
+        long size = rb.getLong();
+        long requestorTime = rb.getLong();
+        long responderTime = rb.getLong();
+        byte[] padding = padding(length);
+        rb.get(padding);
+
+        long endTag = rb.getLong();
+        checkState(endTag == END_TAG, "Incorrect message end");
+
+        return new TestMessage((int) size, requestorTime, responderTime, padding);
     }
     }
 
 
-    /**
-     * {@inheritDoc}
-     * <p/>
-     * This implementation enforces the message length against the buffer
-     * supported length.
-     *
-     * @throws IllegalArgumentException if message size does not match the
-     *                                  supported buffer size
-     */
     @Override
     @Override
     protected void write(TestMessage message, ByteBuffer wb) {
     protected void write(TestMessage message, ByteBuffer wb) {
         if (message.length() != length) {
         if (message.length() != length) {
             throw new IllegalArgumentException(E_WRONG_LEN + message.length());
             throw new IllegalArgumentException(E_WRONG_LEN + message.length());
         }
         }
-        wb.put(message.data());
+
+        wb.putLong(START_TAG);
+        wb.putLong(message.length());
+        wb.putLong(message.requestorTime());
+        wb.putLong(message.responderTime());
+        wb.put(message.padding(), 0, length - META_LENGTH);
+        wb.putLong(END_TAG);
     }
     }
 
 
+    public byte[] padding(int msgLength) {
+        return new byte[msgLength - META_LENGTH];
+    }
 }
 }