Jian Li
Committed by Gerrit Code Review

[ONOS-4176] Implement influxdb retriever for querying metrics

Change-Id: Ia1f3fc4fb3c76fafd003320940b8fe16b039ddae
......@@ -21,5 +21,10 @@
<bundle>mvn:${project.groupId}/onos-app-influxdb/${project.version}</bundle>
<bundle>wrap:mvn:com.izettle/metrics-influxdb/1.1.1$Bundle-SymbolicName=metrics-influxdb&amp;Bundle-Version=1.1.1</bundle>
<bundle>mvn:commons-codec/commons-codec/1.10</bundle>
<bundle>wrap:mvn:org.influxdb/influxdb-java/2.1$Bundle-SymbolicName=influxdb-java&amp;Bundle-Version=2.1</bundle>
<bundle>wrap:mvn:com.squareup.retrofit/retrofit/1.9.0$Bundle-SymbolicName=retrofit&amp;Bundle-Version=1.9.0</bundle>
<bundle>wrap:mvn:com.squareup.okhttp/okhttp/2.4.0$Bundle-SymbolicName=okhttp&amp;Bundle-Version=2.4.0</bundle>
<bundle>wrap:mvn:com.squareup.okio/okio/1.4.0$Bundle-SymbolicName=okio&amp;Bundle-Version=1.4.0</bundle>
<bundle>mvn:com.google.code.gson/gson/2.3.1</bundle>
</feature>
</features>
......
......@@ -71,6 +71,11 @@
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>org.influxdb</groupId>
<artifactId>influxdb-java</artifactId>
<version>2.1</version>
</dependency>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-api</artifactId>
<version>${project.version}</version>
......
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.influxdbmetrics;
import com.google.common.collect.BiMap;
import com.google.common.collect.EnumHashBiMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.lang.StringUtils;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
import org.apache.felix.scr.annotations.Modified;
import org.apache.felix.scr.annotations.Property;
import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.apache.felix.scr.annotations.Service;
import org.influxdb.InfluxDB;
import org.influxdb.InfluxDBFactory;
import org.influxdb.dto.Query;
import org.influxdb.dto.QueryResult;
import org.onlab.util.Tools;
import org.onosproject.cfg.ComponentConfigService;
import org.onosproject.cluster.NodeId;
import org.onosproject.core.CoreService;
import org.osgi.service.component.ComponentContext;
import org.slf4j.Logger;
import java.util.Dictionary;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.slf4j.LoggerFactory.getLogger;
/**
* A Metric retriever implementation for querying metrics from influxDB server.
*/
@Component(immediate = true)
@Service
public class DefaultInfluxDbMetricsRetriever implements InfluxDbMetricsRetriever {
private final Logger log = getLogger(getClass());
private static final String DEFAULT_PROTOCOL = "http";
private static final String DEFAULT_ADDRESS = "localhost";
private static final int DEFAULT_PORT = 8086;
private static final String DEFAULT_DATABASE = "onos";
private static final String DEFAULT_USERNAME = "onos";
private static final String DEFAULT_PASSWORD = "onos.password";
private static final String DEFAULT_POLICY = "default";
private static final String COLON_SEPARATOR = ":";
private static final String SLASH_SEPARATOR = "//";
private static final String BRACKET_START = "[";
private static final String BRACKET_END = "]";
private static final String METRIC_DELIMITER = ".";
private static final int NUB_OF_DELIMITER = 3;
private static final BiMap<TimeUnit, String> TIME_UNIT_MAP =
EnumHashBiMap.create(TimeUnit.class);
static {
// key is TimeUnit enumeration type
// value is influx database time unit keyword
TIME_UNIT_MAP.put(TimeUnit.DAYS, "d");
TIME_UNIT_MAP.put(TimeUnit.HOURS, "h");
TIME_UNIT_MAP.put(TimeUnit.MINUTES, "m");
TIME_UNIT_MAP.put(TimeUnit.SECONDS, "s");
}
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected CoreService coreService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ComponentConfigService cfgService;
@Property(name = "address", value = DEFAULT_ADDRESS,
label = "IP address of influxDB server; " +
"default is localhost")
protected String address = DEFAULT_ADDRESS;
@Property(name = "port", intValue = DEFAULT_PORT,
label = "Port number of influxDB server; " +
"default is 8086")
protected int port = DEFAULT_PORT;
@Property(name = "database", value = DEFAULT_DATABASE,
label = "Database name of influxDB server; " +
"default is onos")
protected String database = DEFAULT_DATABASE;
@Property(name = "username", value = DEFAULT_USERNAME,
label = "Username of influxDB server; default is onos")
protected String username = DEFAULT_USERNAME;
@Property(name = "password", value = DEFAULT_PASSWORD,
label = "Password of influxDB server; default is onos.password")
protected String password = DEFAULT_PASSWORD;
InfluxDB influxDB;
@Activate
public void activate() {
cfgService.registerProperties(getClass());
coreService.registerApplication("org.onosproject.influxdbmetrics");
config();
log.info("Started");
}
@Deactivate
public void deactivate() {
cfgService.unregisterProperties(getClass(), false);
log.info("Stopped");
}
@Modified
public void modified(ComponentContext context) {
readComponentConfiguration(context);
config();
}
private void config() {
StringBuilder url = new StringBuilder();
url.append(DEFAULT_PROTOCOL);
url.append(COLON_SEPARATOR + SLASH_SEPARATOR);
url.append(address);
url.append(COLON_SEPARATOR);
url.append(port);
influxDB = InfluxDBFactory.connect(url.toString(), username, password);
}
@Override
public Map<NodeId, Map<String, List<InfluxMetric>>> allMetrics(int period,
TimeUnit unit) {
Map<NodeId, Set<String>> nameMap = allMetricNames();
Map<NodeId, Map<String, List<InfluxMetric>>> metricsMap = Maps.newHashMap();
nameMap.forEach((nodeId, metricNames) ->
metricsMap.putIfAbsent(nodeId, metricsByNodeId(nodeId, period, unit))
);
return metricsMap;
}
@Override
public Map<String, List<InfluxMetric>> metricsByNodeId(NodeId nodeId, int period,
TimeUnit unit) {
Map<NodeId, Set<String>> nameMap = allMetricNames();
Map<String, List<InfluxMetric>> map = Maps.newHashMap();
nameMap.get(nodeId).forEach(metricName -> {
List<InfluxMetric> value = metric(nodeId, metricName, period, unit);
if (value != null) {
map.putIfAbsent(metricName, value);
}
});
return map;
}
@Override
public Map<NodeId, List<InfluxMetric>> metricsByName(String metricName, int period,
TimeUnit unit) {
Map<NodeId, List<InfluxMetric>> map = Maps.newHashMap();
List<InfluxMetric> metrics = Lists.newArrayList();
String queryPrefix = new StringBuilder()
.append("SELECT m1_rate FROM")
.append(database)
.append(METRIC_DELIMITER)
.append(quote(DEFAULT_POLICY))
.append(METRIC_DELIMITER)
.toString();
String querySuffix = new StringBuilder()
.append(" WHERE time > now() - ")
.append(period)
.append(unitString(unit))
.toString();
allMetricNames().keySet().forEach(nodeId -> {
String queryString = new StringBuilder()
.append(queryPrefix)
.append(quote(nodeId + METRIC_DELIMITER + metricName))
.append(querySuffix)
.toString();
Query query = new Query(queryString, database);
List<QueryResult.Result> results = influxDB.query(query).getResults();
if (results != null && results.get(0) != null
&& results.get(0).getSeries() != null) {
results.get(0).getSeries().get(0).getValues().forEach(value ->
metrics.add(new DefaultInfluxMetric.Builder()
.time((String) value.get(0))
.oneMinRate((Double) value.get(1))
.build()));
map.putIfAbsent(nodeId, metrics);
}
});
return map;
}
@Override
public List<InfluxMetric> metric(NodeId nodeId, String metricName,
int period, TimeUnit unit) {
List<InfluxMetric> metrics = Lists.newArrayList();
String queryString = new StringBuilder()
.append("SELECT m1_rate FROM ")
.append(database)
.append(METRIC_DELIMITER)
.append(quote(DEFAULT_POLICY))
.append(METRIC_DELIMITER)
.append(quote(nodeId + METRIC_DELIMITER + metricName))
.append(" WHERE time > now() - ")
.append(period)
.append(unitString(unit))
.toString();
Query query = new Query(queryString, database);
List<QueryResult.Result> results = influxDB.query(query).getResults();
if (results != null && results.get(0) != null
&& results.get(0).getSeries() != null) {
results.get(0).getSeries().get(0).getValues().forEach(value ->
metrics.add(new DefaultInfluxMetric.Builder()
.time((String) value.get(0))
.oneMinRate((Double) value.get(1))
.build()));
return metrics;
}
return null;
}
@Override
public Map<NodeId, Map<String, InfluxMetric>> allMetrics() {
Map<NodeId, Set<String>> nameMap = allMetricNames();
Map<NodeId, Map<String, InfluxMetric>> metricsMap = Maps.newHashMap();
nameMap.forEach((nodeId, metricNames) ->
metricsMap.putIfAbsent(nodeId, metricsByNodeId(nodeId))
);
return metricsMap;
}
@Override
public Map<String, InfluxMetric> metricsByNodeId(NodeId nodeId) {
Map<NodeId, Set<String>> nameMap = allMetricNames();
Map<String, InfluxMetric> map = Maps.newHashMap();
nameMap.get(nodeId).forEach(metricName -> {
InfluxMetric value = metric(nodeId, metricName);
if (value != null) {
map.putIfAbsent(metricName, value);
}
});
return map;
}
@Override
public Map<NodeId, InfluxMetric> metricsByName(String metricName) {
Map<NodeId, InfluxMetric> map = Maps.newHashMap();
String queryPrefix = new StringBuilder()
.append("SELECT m1_rate FROM")
.append(database)
.append(METRIC_DELIMITER)
.append(quote(DEFAULT_POLICY))
.append(METRIC_DELIMITER)
.toString();
String querySuffix = new StringBuilder()
.append(" LIMIT 1")
.toString();
allMetricNames().keySet().forEach(nodeId -> {
String queryString = new StringBuilder()
.append(queryPrefix)
.append(quote(nodeId + METRIC_DELIMITER + metricName))
.append(querySuffix)
.toString();
Query query = new Query(queryString, database);
List<QueryResult.Result> results = influxDB.query(query).getResults();
if (results != null && results.get(0) != null
&& results.get(0).getSeries() != null) {
InfluxMetric metric = new DefaultInfluxMetric.Builder()
.time((String) results.get(0).getSeries().get(0).getValues().get(0).get(0))
.oneMinRate((Double) results.get(0).getSeries().get(0)
.getValues().get(0).get(1)).build();
map.putIfAbsent(nodeId, metric);
}
});
return map;
}
@Override
public InfluxMetric metric(NodeId nodeId, String metricName) {
String queryString = new StringBuilder()
.append("SELECT m1_rate FROM ")
.append(database)
.append(METRIC_DELIMITER)
.append(quote(DEFAULT_POLICY))
.append(METRIC_DELIMITER)
.append(quote(nodeId + METRIC_DELIMITER + metricName))
.append(" LIMIT 1")
.toString();
Query query = new Query(queryString, database);
List<QueryResult.Result> results = influxDB.query(query).getResults();
if (results != null && results.get(0) != null
&& results.get(0).getSeries() != null) {
return new DefaultInfluxMetric.Builder()
.time((String) results.get(0).getSeries().get(0).getValues().get(0).get(0))
.oneMinRate((Double) results.get(0).getSeries().get(0)
.getValues().get(0).get(1)).build();
}
return null;
}
private String unitString(TimeUnit unit) {
return TIME_UNIT_MAP.get(unit) == null ? "h" : TIME_UNIT_MAP.get(unit);
}
private String quote(String str) {
return "\"" + str + "\"";
}
/**
* Returns all metric names that bound with node identification.
*
* @return all metric names
*/
protected Map<NodeId, Set<String>> allMetricNames() {
Map<NodeId, Set<String>> metricNameMap = Maps.newHashMap();
Query query = new Query("SHOW MEASUREMENTS", database);
List<QueryResult.Result> results = influxDB.query(query).getResults();
List<List<Object>> rawMetricNames = results.get(0).getSeries().get(0).getValues();
rawMetricNames.forEach(rawMetricName -> {
String nodeIdStr = getNodeId(strip(rawMetricName.toString()));
if (nodeIdStr != null) {
NodeId nodeId = NodeId.nodeId(nodeIdStr);
String metricName = getMetricName(strip(rawMetricName.toString()));
if (!metricNameMap.containsKey(nodeId)) {
metricNameMap.putIfAbsent(nodeId, Sets.newHashSet());
}
if (metricName != null) {
metricNameMap.get(nodeId).add(metricName);
}
}
});
return metricNameMap;
}
/**
* Strips special bracket from the full name.
*
* @param fullName full name
* @return bracket stripped string
*/
private String strip(String fullName) {
return StringUtils.strip(StringUtils.strip(fullName, BRACKET_START), BRACKET_END);
}
/**
* Returns metric name from full name.
* The elements in full name is split by using '.';
* We assume that the metric name always comes after the last three '.'
*
* @param fullName full name
* @return metric name
*/
private String getMetricName(String fullName) {
int index = StringUtils.lastOrdinalIndexOf(fullName,
METRIC_DELIMITER, NUB_OF_DELIMITER);
if (index != -1) {
return StringUtils.substring(fullName, index + 1);
} else {
log.warn("Database {} contains malformed metric name.", database);
return null;
}
}
/**
* Returns node id from full name.
* The elements in full name is split by using '.';
* We assume that the node id always comes before the last three '.'
*
* @param fullName full name
* @return node id
*/
private String getNodeId(String fullName) {
int index = StringUtils.lastOrdinalIndexOf(fullName,
METRIC_DELIMITER, NUB_OF_DELIMITER);
if (index != -1) {
return StringUtils.substring(fullName, 0, index);
} else {
log.warn("Database {} contains malformed node id.", database);
return null;
}
}
/**
* Extracts properties from the component configuration context.
*
* @param context the component context
*/
private void readComponentConfiguration(ComponentContext context) {
Dictionary<?, ?> properties = context.getProperties();
String addressStr = Tools.get(properties, "address");
address = addressStr != null ? addressStr : DEFAULT_ADDRESS;
log.info("Configured. InfluxDB server address is {}", address);
String databaseStr = Tools.get(properties, "database");
database = databaseStr != null ? databaseStr : DEFAULT_DATABASE;
log.info("Configured. InfluxDB server database is {}", database);
String usernameStr = Tools.get(properties, "username");
username = usernameStr != null ? usernameStr : DEFAULT_USERNAME;
log.info("Configured. InfluxDB server username is {}", username);
String passwordStr = Tools.get(properties, "password");
password = passwordStr != null ? passwordStr : DEFAULT_PASSWORD;
log.info("Configured. InfluxDB server password is {}", password);
Integer portConfigured = Tools.getIntegerProperty(properties, "port");
if (portConfigured == null) {
port = DEFAULT_PORT;
log.info("InfluxDB port is not configured, default value is {}", port);
} else {
port = portConfigured;
log.info("Configured. InfluxDB port is configured to {}", port);
}
}
}
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.influxdbmetrics;
import org.apache.commons.lang.StringUtils;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import static com.google.common.base.Preconditions.checkNotNull;
/**
* Default implementation of influx metric.
*/
public final class DefaultInfluxMetric implements InfluxMetric {
private double oneMinRate;
private DateTime time;
private DefaultInfluxMetric(double oneMinRate, DateTime time) {
this.oneMinRate = oneMinRate;
this.time = time;
}
@Override
public double oneMinRate() {
return oneMinRate;
}
@Override
public DateTime time() {
return time;
}
public static final class Builder implements InfluxMetric.Builder {
private double oneMinRate;
private String timestamp;
private static final String TIMESTAMP_MSG = "Must specify a timestamp.";
private static final String ONE_MIN_RATE_MSG = "Must specify one minute rate.";
public Builder() {
}
@Override
public InfluxMetric.Builder oneMinRate(double rate) {
this.oneMinRate = rate;
return this;
}
@Override
public InfluxMetric.Builder time(String time) {
this.timestamp = time;
return this;
}
@Override
public InfluxMetric build() {
checkNotNull(oneMinRate, ONE_MIN_RATE_MSG);
checkNotNull(timestamp, TIMESTAMP_MSG);
return new DefaultInfluxMetric(oneMinRate, parseTime(timestamp));
}
private DateTime parseTime(String time) {
String reformatTime = StringUtils.replace(StringUtils.replace(time, "T", " "), "Z", "");
DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
return formatter.parseDateTime(reformatTime);
}
}
}
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.influxdbmetrics;
import org.onosproject.cluster.NodeId;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* A Metric retriever interface for querying metrics value from influxDB server.
*/
public interface InfluxDbMetricsRetriever {
/**
* Returns last metric values from all nodes.
*
* @return all metrics from all nodes
*/
Map<NodeId, Map<String, InfluxMetric>> allMetrics();
/**
* Returns last metric values from a node.
*
* @param nodeId node identification
* @return all metrics from a given node
*/
Map<String, InfluxMetric> metricsByNodeId(NodeId nodeId);
/**
* Returns a collection of last metric values from all nodes.
*
* @param metricName metric name
* @return a collection of metrics from all nodes
*/
Map<NodeId, InfluxMetric> metricsByName(String metricName);
/**
* Returns a last metric value from a given node.
*
* @param nodeId node identification
* @param metricName metric name
* @return a metric value from a given node
*/
InfluxMetric metric(NodeId nodeId, String metricName);
/**
* Returns metric values of all nodes within a given period of time.
*
* @param period projected period
* @param unit time unit
* @return all metric values of all nodes
*/
Map<NodeId, Map<String, List<InfluxMetric>>> allMetrics(int period, TimeUnit unit);
/**
* Returns metric values of a node within a given period of time.
*
* @param nodeId node identification
* @param period projected period
* @param unit time unit
* @return metric value of a node
*/
Map<String, List<InfluxMetric>> metricsByNodeId(NodeId nodeId, int period, TimeUnit unit);
/**
* Returns a collection of last metric values of all nodes within a given period of time.
*
* @param metricName metric name
* @param period projected period
* @param unit time unit
* @return metric value of all nodes
*/
Map<NodeId, List<InfluxMetric>> metricsByName(String metricName, int period, TimeUnit unit);
/**
* Returns metric value of a given node within a given period of time.
*
* @param nodeId node identification
* @param metricName metric name
* @param period projected period
* @param unit time unit
* @return metric value of a node
*/
List<InfluxMetric> metric(NodeId nodeId, String metricName, int period, TimeUnit unit);
}
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.influxdbmetrics;
import org.joda.time.DateTime;
/**
* Metric that represents all values queried from influx database.
*/
public interface InfluxMetric {
/**
* Returns one minute rate of the given metric.
*
* @return one minute rate of the given metric
*/
double oneMinRate();
/**
* Returns collected timestamp of the given metric.
*
* @return collected timestamp of the given metric
*/
DateTime time();
/**
* A builder of InfluxMetric.
*/
interface Builder {
/**
* Sets one minute rate.
*
* @param rate one minute rate
* @return builder object
*/
Builder oneMinRate(double rate);
/**
* Sets collected timestamp.
*
* @param time timestamp
* @return builder object
*/
Builder time(String time);
/**
* Builds a influx metric instance.
*
* @return influx metric instance
*/
InfluxMetric build();
}
}