Added initial implementation of Topology-related event and

event metrics collector. It can be loaded by one of the following two (new)
features: onos-app-metrics, onos-app-metrics-topology

After loading the module, it subscribes for topology-related events
and keeps the following state:
 (a) The last 10 events
 (b) The timestamp of the last event (ms after epoch) as observed by this
     module
 (c) The rate of the topology events: count, median rate, average rate
     over the last 1, 5 or 15 minutes

The following CLI commands are added:
 * onos:topology-events
   Shows the last 10 topology events

 * onos:topology-events-metrics
   Shows the timestamp of the last event, and the rate of the topology
   events: see (b) and (c) above

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.onlab.onos</groupId>
+    <artifactId>onos-apps</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>onos-app-metrics</artifactId>
+  <packaging>pom</packaging>
+
+  <description>ONOS metrics applications</description>
+
+  <modules>
+    <module>topology</module>
+  </modules>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.onlab.onos</groupId>
+      <artifactId>onlab-misc</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+    </dependency>
+
+  </dependencies>
+
+</project>

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.onlab.onos</groupId>
+    <artifactId>onos-app-metrics</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>onos-app-metrics-topology</artifactId>
+  <packaging>bundle</packaging>
+
+  <description>ONOS topology metrics application</description>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.onlab.onos</groupId>
+      <artifactId>onos-cli</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.karaf.shell</groupId>
+      <artifactId>org.apache.karaf.shell.console</artifactId>
+    </dependency>
+  </dependencies>
+
+</project>

+package org.onlab.onos.metrics.topology;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Meter;
+import com.google.common.collect.ImmutableList;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.metrics.MetricsComponent;
+import org.onlab.metrics.MetricsFeature;
+import org.onlab.metrics.MetricsService;
+import org.onlab.onos.event.Event;
+import org.onlab.onos.net.topology.TopologyEvent;
+import org.onlab.onos.net.topology.TopologyListener;
+import org.onlab.onos.net.topology.TopologyService;
+import org.slf4j.Logger;
+
+/**
+ * ONOS Topology Metrics Application that collects topology-related metrics.
+ */
+@Component(immediate = true)
+@Service
+public class TopologyMetrics implements TopologyMetricsService,
+                                        TopologyListener {
+    private static final Logger log = getLogger(TopologyMetrics.class);
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected TopologyService topologyService;
+    private LinkedList<TopologyEvent> lastEvents = new LinkedList<>();
+    private static final int LAST_EVENTS_MAX_N = 10;
+
+    //
+    // Metrics
+    //
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected MetricsService metricsService;
+    //
+    private static final String COMPONENT_NAME = "Topology";
+    private static final String FEATURE_NAME = "EventNotification";
+    private static final String GAUGE_NAME = "LastEventTimestamp.EpochMs";
+    private static final String METER_NAME = "EventRate";
+    //
+    private MetricsComponent metricsComponent;
+    private MetricsFeature metricsFeatureEventNotification;
+    //
+    // Timestamp of the last Topology event (ms from the Epoch)
+    private volatile long lastEventTimestampEpochMs = 0;
+    private Gauge<Long> lastEventTimestampEpochMsGauge;
+    // Rate of the Topology events published to the Topology listeners
+    private Meter eventRateMeter;
+
+    @Activate
+    protected void activate() {
+        clear();
+        registerMetrics();
+        topologyService.addListener(this);
+        log.info("ONOS Topology Metrics started.");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        topologyService.removeListener(this);
+        removeMetrics();
+        clear();
+        log.info("ONOS Topology Metrics stopped.");
+    }
+
+    @Override
+    public List<TopologyEvent> getEvents() {
+        synchronized (lastEvents) {
+            return ImmutableList.<TopologyEvent>copyOf(lastEvents);
+        }
+    }
+
+    @Override
+    public Gauge<Long> lastEventTimestampEpochMsGauge() {
+        return lastEventTimestampEpochMsGauge;
+    }
+
+    @Override
+    public Meter eventRateMeter() {
+        return eventRateMeter;
+    }
+
+    @Override
+    public void event(TopologyEvent event) {
+        lastEventTimestampEpochMs = System.currentTimeMillis();
+        //
+        // NOTE: If we want to count each "reason" as a separate event,
+        // then we should use 'event.reason().size()' instead of '1' to
+        // mark the meter below.
+        //
+        eventRateMeter.mark(1);
+
+        log.debug("Topology Event: time = {} type = {} subject = {}",
+                  event.time(), event.type(), event.subject());
+        for (Event reason : event.reasons()) {
+            log.debug("Topology Event Reason: time = {} type = {} subject = {}",
+                      reason.time(), reason.type(), reason.subject());
+        }
+
+        //
+        // Keep only the last N events, where N = LAST_EVENTS_MAX_N
+        //
+        synchronized (lastEvents) {
+            while (lastEvents.size() >= LAST_EVENTS_MAX_N) {
+                lastEvents.remove();
+            }
+            lastEvents.add(event);
+        }
+    }
+
+    /**
+     * Clears the internal state.
+     */
+    private void clear() {
+        lastEventTimestampEpochMs = 0;
+        synchronized (lastEvents) {
+            lastEvents.clear();
+        }
+    }
+
+    /**
+     * Registers the metrics.
+     */
+    private void registerMetrics() {
+        metricsComponent = metricsService.registerComponent(COMPONENT_NAME);
+        metricsFeatureEventNotification =
+            metricsComponent.registerFeature(FEATURE_NAME);
+        lastEventTimestampEpochMsGauge =
+            metricsService.registerMetric(metricsComponent,
+                                          metricsFeatureEventNotification,
+                                          GAUGE_NAME,
+                                          new Gauge<Long>() {
+                                              @Override
+                                              public Long getValue() {
+                                                  return lastEventTimestampEpochMs;
+                                              }
+                                          });
+        eventRateMeter =
+            metricsService.createMeter(metricsComponent,
+                                       metricsFeatureEventNotification,
+                                       METER_NAME);
+
+    }
+
+    /**
+     * Removes the metrics.
+     */
+    private void removeMetrics() {
+        metricsService.removeMetric(metricsComponent,
+                                    metricsFeatureEventNotification,
+                                    GAUGE_NAME);
+        metricsService.removeMetric(metricsComponent,
+                                    metricsFeatureEventNotification,
+                                    METER_NAME);
+    }
+}

+package org.onlab.onos.metrics.topology;
+
+import java.util.List;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Meter;
+import org.onlab.onos.net.topology.TopologyEvent;
+
+/**
+ * Service interface exported by TopologyMetrics.
+ */
+public interface TopologyMetricsService {
+    /**
+     * Gets the last saved topology events.
+     *
+     * @return the last saved topology events.
+     */
+    public List<TopologyEvent> getEvents();
+
+    /**
+     * Gets the Metrics' Gauge for the last topology event timestamp
+     * (ms from the epoch).
+     *
+     * @return the Metrics' Gauge for the last topology event timestamp
+     * (ms from the epoch)
+     */
+    public Gauge<Long> lastEventTimestampEpochMsGauge();
+
+    /**
+     * Gets the Metrics' Meter for the topology events rate.
+     *
+     * @return the Metrics' Meter for the topology events rate
+     */
+    public Meter eventRateMeter();
+}

+package org.onlab.onos.metrics.topology.cli;
+
+import java.util.List;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.cli.AbstractShellCommand;
+import org.onlab.onos.event.Event;
+import org.onlab.onos.metrics.topology.TopologyMetricsService;
+import org.onlab.onos.net.topology.TopologyEvent;
+
+/**
+ * Command to show the list of last topology events.
+ */
+@Command(scope = "onos", name = "topology-events",
+         description = "Lists the last topology events")
+public class TopologyEventsListCommand extends AbstractShellCommand {
+
+    private static final String FORMAT_EVENT =
+        "Topology Event time=%d type=%s subject=%s";
+    private static final String FORMAT_REASON =
+        "    Reason time=%d type=%s subject=%s";
+
+    @Override
+    protected void execute() {
+        TopologyMetricsService service = get(TopologyMetricsService.class);
+
+        if (outputJson()) {
+            print("%s", json(service.getEvents()));
+        } else {
+            for (TopologyEvent event : service.getEvents()) {
+                print(FORMAT_EVENT, event.time(), event.type(),
+                      event.subject());
+                for (Event reason : event.reasons()) {
+                    print(FORMAT_REASON, reason.time(), reason.type(),
+                          reason.subject());
+                }
+                print("");          // Extra empty line for clarity
+            }
+        }
+    }
+
+    /**
+     * Produces a JSON array of topology events.
+     *
+     * @param topologyEvents the topology events with the data
+     * @return JSON array with the topology events
+     */
+    private JsonNode json(List<TopologyEvent> topologyEvents) {
+        ObjectMapper mapper = new ObjectMapper();
+        ArrayNode result = mapper.createArrayNode();
+
+        for (TopologyEvent event : topologyEvents) {
+            result.add(json(mapper, event));
+        }
+        return result;
+    }
+
+    /**
+     * Produces JSON object for a topology event.
+     *
+     * @param mapper the JSON object mapper to use
+     * @param topologyEvent the topology event with the data
+     * @return JSON object for the topology event
+     */
+    private ObjectNode json(ObjectMapper mapper, TopologyEvent topologyEvent) {
+        ObjectNode result = mapper.createObjectNode();
+        ArrayNode reasons = mapper.createArrayNode();
+
+        for (Event reason : topologyEvent.reasons()) {
+            reasons.add(json(mapper, reason));
+        }
+        result.put("time", topologyEvent.time())
+            .put("type", topologyEvent.type().toString())
+            .put("subject", topologyEvent.subject().toString())
+            .put("reasons", reasons);
+        return result;
+    }
+
+    /**
+     * Produces JSON object for a generic event.
+     *
+     * @param event the generic event with the data
+     * @return JSON object for the generic event
+     */
+    private ObjectNode json(ObjectMapper mapper, Event event) {
+        ObjectNode result = mapper.createObjectNode();
+
+        result.put("time", event.time())
+            .put("type", event.type().toString())
+            .put("subject", event.subject().toString());
+        return result;
+    }
+}

+package org.onlab.onos.metrics.topology.cli;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.json.MetricsModule;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.cli.AbstractShellCommand;
+import org.onlab.onos.metrics.topology.TopologyMetricsService;
+
+/**
+ * Command to show the topology events metrics.
+ */
+@Command(scope = "onos", name = "topology-events-metrics",
+         description = "Lists topology events metrics")
+public class TopologyEventsMetricsCommand extends AbstractShellCommand {
+
+    private static final String FORMAT_GAUGE =
+        "Last Topology Event Timestamp (ms from epoch)=%d";
+    private static final String FORMAT_METER =
+        "Topology Events count=%d rate(events/sec) mean=%f m1=%f m5=%f m15=%f";
+
+    @Override
+    protected void execute() {
+        TopologyMetricsService service = get(TopologyMetricsService.class);
+        Gauge<Long> gauge = service.lastEventTimestampEpochMsGauge();
+        Meter meter = service.eventRateMeter();
+
+        if (outputJson()) {
+            ObjectMapper mapper = new ObjectMapper()
+                .registerModule(new MetricsModule(TimeUnit.SECONDS,
+                                                  TimeUnit.MILLISECONDS,
+                                                  false));
+            ObjectNode result = mapper.createObjectNode();
+            try {
+                //
+                // NOTE: The API for custom serializers is incomplete,
+                // hence we have to parse the JSON string to create JsonNode.
+                //
+                final String gaugeJson = mapper.writeValueAsString(gauge);
+                final String meterJson = mapper.writeValueAsString(meter);
+                JsonNode gaugeNode = mapper.readTree(gaugeJson);
+                JsonNode meterNode = mapper.readTree(meterJson);
+                result.put("lastTopologyEventTimestamp", gaugeNode);
+                result.put("listenerEventRate", meterNode);
+            } catch (JsonProcessingException e) {
+                log.error("Error writing value as JSON string", e);
+            } catch (IOException e) {
+                log.error("Error writing value as JSON string", e);
+            }
+            print("%s", result);
+        } else {
+            TimeUnit rateUnit = TimeUnit.SECONDS;
+            double rateFactor = rateUnit.toSeconds(1);
+            print(FORMAT_GAUGE, gauge.getValue());
+            print(FORMAT_METER, meter.getCount(),
+                  meter.getMeanRate() * rateFactor,
+                  meter.getOneMinuteRate() * rateFactor,
+                  meter.getFiveMinuteRate() * rateFactor,
+                  meter.getFifteenMinuteRate() * rateFactor);
+        }
+    }
+}

+/**
+ * ONOS Topology Metrics Application that collects topology-related metrics.
+ */
+package org.onlab.onos.metrics.topology;

+<blueprint xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0">
+
+  <command-bundle xmlns="http://karaf.apache.org/xmlns/shell/v1.1.0">
+    <command>
+      <action class="org.onlab.onos.metrics.topology.cli.TopologyEventsListCommand"/>
+    </command>
+    <command>
+      <action class="org.onlab.onos.metrics.topology.cli.TopologyEventsMetricsCommand"/>
+    </command>
+  </command-bundle>
+
+</blueprint>

@@ -27,6 +27,7 @@
         <module>sdnip</module>
         <module>calendar</module>
         <module>optical</module>
+        <module>metrics</module>
     </modules>
 
     <properties>

@@ -37,6 +37,7 @@
 
         <bundle>mvn:com.hazelcast/hazelcast/3.3</bundle>
         <bundle>mvn:io.dropwizard.metrics/metrics-core/3.1.0</bundle>
+        <bundle>mvn:io.dropwizard.metrics/metrics-json/3.1.0</bundle>
         <bundle>mvn:com.eclipsesource.minimal-json/minimal-json/0.9.1</bundle>
 
         <bundle>mvn:com.esotericsoftware/kryo/3.0.0</bundle>
@@ -183,7 +184,6 @@
         <bundle>mvn:org.onlab.onos/onos-app-optical/1.0.0-SNAPSHOT</bundle>
      </feature>
 
-
     <feature name="onos-app-sdnip" version="1.0.0"
              description="SDN-IP peering application">
         <feature>onos-api</feature>
@@ -197,4 +197,15 @@
         <bundle>mvn:org.onlab.onos/onos-app-calendar/1.0.0-SNAPSHOT</bundle>
     </feature>
 
+    <feature name="onos-app-metrics" version="1.0.0"
+             description="ONOS metrics applications">
+        <feature>onos-app-metrics-topology</feature>
+    </feature>
+
+    <feature name="onos-app-metrics-topology" version="1.0.0"
+             description="ONOS topology metrics application">
+        <feature>onos-api</feature>
+        <bundle>mvn:org.onlab.onos/onos-app-metrics-topology/1.0.0-SNAPSHOT</bundle>
+    </feature>
+
 </features>

@@ -53,6 +53,11 @@
             <version>3.1.0</version>
         </dependency>
         <dependency>
+            <groupId>io.dropwizard.metrics</groupId>
+            <artifactId>metrics-json</artifactId>
+            <version>3.1.0</version>
+        </dependency>
+        <dependency>
             <groupId>org.apache.felix</groupId>
             <artifactId>org.apache.felix.scr.annotations</artifactId>
         </dependency>