Madan Jampani
Committed by Gerrit Code Review

Support for a distributed counter

Change-Id: I346e9baa28556fac13e53771021f5f6fbcd75ac9
Showing 21 changed files with 587 additions and 39 deletions
/*
* Copyright 2015 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.cli.net;
import java.util.Map;
import org.apache.karaf.shell.commands.Command;
import org.onosproject.cli.AbstractShellCommand;
import org.onosproject.store.service.StorageAdminService;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* Command to list the various counters in the system.
*/
@Command(scope = "onos", name = "counters",
description = "Lists information about atomic counters in the system")
public class CountersListCommand extends AbstractShellCommand {
private static final String FMT = "name=%s next_value=%d";
/**
* Displays counters as text.
*
* @param mapInfo map descriptions
*/
private void displayCounters(Map<String, Long> counters) {
counters.forEach((name, nextValue) -> print(FMT, name, nextValue));
}
/**
* Converts info for counters into a JSON object.
*
* @param counters counter info
*/
private JsonNode json(Map<String, Long> counters) {
ObjectMapper mapper = new ObjectMapper();
ArrayNode jsonCounters = mapper.createArrayNode();
// Create a JSON node for each counter
counters.forEach((name, value) -> {
ObjectNode jsonCounter = mapper.createObjectNode();
jsonCounter.put("name", name)
.put("value", value);
jsonCounters.add(jsonCounter);
});
return jsonCounters;
}
@Override
protected void execute() {
StorageAdminService storageAdminService = get(StorageAdminService.class);
Map<String, Long> counters = storageAdminService.getCounters();
if (outputJson()) {
print("%s", json(counters));
} else {
displayCounters(counters);
}
}
}
......@@ -233,6 +233,9 @@
<action class="org.onosproject.cli.net.MapsListCommand"/>
</command>
<command>
<action class="org.onosproject.cli.net.CountersListCommand"/>
</command>
<command>
<action class="org.onosproject.cli.net.TransactionsCommand"/>
</command>
<command>
......
/*
* Copyright 2015 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.store.service;
import java.util.concurrent.CompletableFuture;
/**
* An async atomic counter dispenses monotonically increasing values.
*/
public interface AsyncAtomicCounter {
/**
* Atomically increment by one the current value.
*
* @return updated value
*/
CompletableFuture<Long> incrementAndGet();
/**
* Returns the current value of the counter without modifying it.
*
* @return current value
*/
CompletableFuture<Long> get();
}
/*
* Copyright 2015 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.store.service;
/**
* An atomic counter dispenses monotonically increasing values.
*/
public interface AtomicCounter {
/**
* Atomically increment by one the current value.
*
* @return updated value
*/
long incrementAndGet();
/**
* Returns the current value of the counter without modifying it.
*
* @return current value
*/
long get();
}
package org.onosproject.store.service;
/**
* Builder for AtomicCounter.
*/
public interface AtomicCounterBuilder {
/**
* Sets the name for the atomic counter.
* <p>
* Each atomic counter is identified by a unique name.
* </p>
* <p>
* Note: This is a mandatory parameter.
* </p>
*
* @param name name of the atomic counter
* @return this AtomicCounterBuilder
*/
public AtomicCounterBuilder withName(String name);
/**
* Creates this counter on the partition that spans the entire cluster.
* <p>
* When partitioning is disabled, the counter state will be
* ephemeral and does not survive a full cluster restart.
* </p>
* <p>
* Note: By default partitions are enabled.
* </p>
* @return this AtomicCounterBuilder
*/
public AtomicCounterBuilder withPartitionsDisabled();
/**
* Builds a AtomicCounter based on the configuration options
* supplied to this builder.
*
* @return new AtomicCounter
* @throws java.lang.RuntimeException if a mandatory parameter is missing
*/
public AtomicCounter build();
/**
* Builds a AsyncAtomicCounter based on the configuration options
* supplied to this builder.
*
* @return new AsyncAtomicCounter
* @throws java.lang.RuntimeException if a mandatory parameter is missing
*/
public AsyncAtomicCounter buildAsyncCounter();
}
......@@ -20,7 +20,7 @@ package org.onosproject.store.service;
* Top level exception for ConsistentMap failures.
*/
@SuppressWarnings("serial")
public class ConsistentMapException extends RuntimeException {
public class ConsistentMapException extends StorageException {
public ConsistentMapException() {
}
......
......@@ -16,6 +16,8 @@
package org.onosproject.store.service;
import org.onlab.util.KryoNamespace;
/**
* Interface for serialization for store artifacts.
*/
......@@ -35,4 +37,24 @@ public interface Serializer {
* @param <T> decoded type
*/
<T> T decode(byte[] bytes);
/**
* Creates a new Serializer instance from a KryoNamespace.
*
* @param kryo kryo namespace
* @return Serializer instance
*/
public static Serializer using(KryoNamespace kryo) {
return new Serializer() {
@Override
public <T> byte[] encode(T object) {
return kryo.serialize(object);
}
@Override
public <T> T decode(byte[] bytes) {
return kryo.deserialize(bytes);
}
};
}
}
\ No newline at end of file
......
......@@ -17,6 +17,7 @@ package org.onosproject.store.service;
import java.util.Collection;
import java.util.List;
import java.util.Map;
/**
* Service for administering storage instances.
......@@ -38,6 +39,13 @@ public interface StorageAdminService {
List<MapInfo> getMapInfo();
/**
* Returns information about all the atomic counters in the system.
*
* @return mapping from counter name to that counter's next value
*/
Map<String, Long> getCounters();
/**
* Returns all the transactions in the system.
*
* @return collection of transactions
......
/*
* Copyright 2015 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.store.service;
/**
* Top level exception for Store failures.
*/
@SuppressWarnings("serial")
public class StorageException extends RuntimeException {
public StorageException() {
}
public StorageException(Throwable t) {
super(t);
}
/**
* Store operation timeout.
*/
public static class Timeout extends StorageException {
}
/**
* Store update conflicts with an in flight transaction.
*/
public static class ConcurrentModification extends StorageException {
}
/**
* Store operation interrupted.
*/
public static class Interrupted extends StorageException {
}
}
......@@ -16,6 +16,7 @@
package org.onosproject.store.service;
/**
* Storage service.
* <p>
......@@ -55,6 +56,13 @@ public interface StorageService {
<E> SetBuilder<E> setBuilder();
/**
* Creates a new AtomicCounterBuilder.
*
* @return atomic counter builder
*/
AtomicCounterBuilder atomicCounterBuilder();
/**
* Creates a new transaction context.
*
* @return transaction context
......
......@@ -18,6 +18,7 @@ package org.onosproject.store.consistent.impl;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import net.kuujo.copycat.CopycatConfig;
......@@ -47,6 +48,7 @@ import org.onosproject.store.cluster.impl.DistributedClusterStore;
import org.onosproject.store.cluster.impl.NodeInfo;
import org.onosproject.store.cluster.messaging.ClusterCommunicationService;
import org.onosproject.store.ecmap.EventuallyConsistentMapBuilderImpl;
import org.onosproject.store.service.AtomicCounterBuilder;
import org.onosproject.store.service.ConsistentMapBuilder;
import org.onosproject.store.service.ConsistentMapException;
import org.onosproject.store.service.EventuallyConsistentMapBuilder;
......@@ -324,6 +326,11 @@ public class DatabaseManager implements StorageService, StorageAdminService {
}
@Override
public AtomicCounterBuilder atomicCounterBuilder() {
return new DefaultAtomicCounterBuilder(inMemoryDatabase, partitionedDatabase);
}
@Override
public List<MapInfo> getMapInfo() {
List<MapInfo> maps = Lists.newArrayList();
maps.addAll(getMapInfo(inMemoryDatabase));
......@@ -339,6 +346,15 @@ public class DatabaseManager implements StorageService, StorageAdminService {
.collect(Collectors.toList());
}
@Override
public Map<String, Long> getCounters() {
Map<String, Long> counters = Maps.newHashMap();
counters.putAll(complete(inMemoryDatabase.counters()));
counters.putAll(complete(partitionedDatabase.counters()));
return counters;
}
@Override
public Collection<Transaction> getTransactions() {
return complete(transactionManager.getTransactions());
......@@ -361,4 +377,4 @@ public class DatabaseManager implements StorageService, StorageAdminService {
public void redriveTransactions() {
getTransactions().stream().forEach(transactionManager::execute);
}
}
}
\ No newline at end of file
......
......@@ -35,6 +35,12 @@ public interface DatabaseProxy<K, V> {
*/
CompletableFuture<Set<String>> tableNames();
/**
* Returns a mapping from counter name to next value.
* @return A completable future to be completed with the result once complete.
*/
CompletableFuture<Map<String, Long>> counters();
/**
* Gets the table size.
*
......@@ -182,6 +188,23 @@ public interface DatabaseProxy<K, V> {
CompletableFuture<Result<Boolean>> replace(String tableName, K key, long oldVersion, V newValue);
/**
* Returns the next value for the specified atomic counter after
* incrementing the current value by one.
*
* @param counterName counter name
* @return next value for the specified counter
*/
CompletableFuture<Long> nextValue(String counterName);
/**
* Returns the current value for the specified atomic counter.
*
* @param counterName counter name
* @return current value for the specified counter
*/
CompletableFuture<Long> currentValue(String counterName);
/**
* Prepare and commit the specified transaction.
*
* @param transaction transaction to commit (after preparation)
......
......@@ -17,6 +17,7 @@
package org.onosproject.store.consistent.impl;
import java.util.Collection;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
......@@ -46,6 +47,9 @@ public interface DatabaseState<K, V> {
Set<String> tableNames();
@Query
Map<String, Long> counters();
@Query
int size(String tableName);
@Query
......@@ -94,6 +98,12 @@ public interface DatabaseState<K, V> {
Result<Boolean> replace(String tableName, K key, long oldVersion, V newValue);
@Command
Long nextValue(String counterName);
@Query
Long currentValue(String counterName);
@Command
boolean prepareAndCommit(Transaction transaction);
@Command
......
/*
* Copyright 2015 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.store.consistent.impl;
import java.util.concurrent.CompletableFuture;
import org.onosproject.store.service.AsyncAtomicCounter;
import static com.google.common.base.Preconditions.*;
/**
* Default implementation for a distributed AsyncAtomicCounter backed by
* partitioned Raft DB.
* <p>
* The initial value will be zero.
*/
public class DefaultAsyncAtomicCounter implements AsyncAtomicCounter {
private final String name;
private final Database database;
public DefaultAsyncAtomicCounter(String name, Database database) {
this.name = checkNotNull(name);
this.database = checkNotNull(database);
}
@Override
public CompletableFuture<Long> incrementAndGet() {
return database.nextValue(name);
}
@Override
public CompletableFuture<Long> get() {
return database.currentValue(name);
}
}
/*
* Copyright 2015 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.store.consistent.impl;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.onosproject.store.service.AsyncAtomicCounter;
import org.onosproject.store.service.AtomicCounter;
import org.onosproject.store.service.StorageException;
/**
* Default implementation for a distributed AtomicCounter backed by
* partitioned Raft DB.
* <p>
* The initial value will be zero.
*/
public class DefaultAtomicCounter implements AtomicCounter {
private static final int OPERATION_TIMEOUT_MILLIS = 5000;
private final AsyncAtomicCounter asyncCounter;
public DefaultAtomicCounter(String name, Database database) {
asyncCounter = new DefaultAsyncAtomicCounter(name, database);
}
@Override
public long incrementAndGet() {
return complete(asyncCounter.incrementAndGet());
}
@Override
public long get() {
return complete(asyncCounter.get());
}
private static <T> T complete(CompletableFuture<T> future) {
try {
return future.get(OPERATION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new StorageException.Interrupted();
} catch (TimeoutException e) {
throw new StorageException.Timeout();
} catch (ExecutionException e) {
throw new StorageException(e.getCause());
}
}
}
package org.onosproject.store.consistent.impl;
import org.onosproject.store.service.AsyncAtomicCounter;
import org.onosproject.store.service.AtomicCounter;
import org.onosproject.store.service.AtomicCounterBuilder;
import static com.google.common.base.Preconditions.checkArgument;
/**
* Default implementation of AtomicCounterBuilder.
*/
public class DefaultAtomicCounterBuilder implements AtomicCounterBuilder {
private String name;
private boolean partitionsEnabled = true;
private final Database partitionedDatabase;
private final Database inMemoryDatabase;
public DefaultAtomicCounterBuilder(Database inMemoryDatabase, Database partitionedDatabase) {
this.inMemoryDatabase = inMemoryDatabase;
this.partitionedDatabase = partitionedDatabase;
}
@Override
public AtomicCounterBuilder withName(String name) {
checkArgument(name != null && !name.isEmpty());
this.name = name;
return this;
}
@Override
public AtomicCounterBuilder withPartitionsDisabled() {
partitionsEnabled = false;
return this;
}
@Override
public AtomicCounter build() {
Database database = partitionsEnabled ? partitionedDatabase : inMemoryDatabase;
return new DefaultAtomicCounter(name, database);
}
@Override
public AsyncAtomicCounter buildAsyncCounter() {
Database database = partitionsEnabled ? partitionedDatabase : inMemoryDatabase;
return new DefaultAsyncAtomicCounter(name, database);
}
}
......@@ -65,6 +65,11 @@ public class DefaultDatabase extends AbstractResource<Database> implements Datab
}
@Override
public CompletableFuture<Map<String, Long>> counters() {
return checkOpen(() -> proxy.counters());
}
@Override
public CompletableFuture<Integer> size(String tableName) {
return checkOpen(() -> proxy.size(tableName));
}
......@@ -145,6 +150,16 @@ public class DefaultDatabase extends AbstractResource<Database> implements Datab
}
@Override
public CompletableFuture<Long> nextValue(String counterName) {
return checkOpen(() -> proxy.nextValue(counterName));
}
@Override
public CompletableFuture<Long> currentValue(String counterName) {
return checkOpen(() -> proxy.currentValue(counterName));
}
@Override
public CompletableFuture<Boolean> prepareAndCommit(Transaction transaction) {
return checkOpen(() -> proxy.prepareAndCommit(transaction));
}
......
......@@ -21,6 +21,7 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.Set;
......@@ -43,6 +44,7 @@ import net.kuujo.copycat.state.StateContext;
*/
public class DefaultDatabaseState implements DatabaseState<String, byte[]> {
private Long nextVersion;
private Map<String, AtomicLong> counters;
private Map<String, Map<String, Versioned<byte[]>>> tables;
/**
......@@ -60,6 +62,11 @@ public class DefaultDatabaseState implements DatabaseState<String, byte[]> {
@Initializer
@Override
public void init(StateContext<DatabaseState<String, byte[]>> context) {
counters = context.get("counters");
if (counters == null) {
counters = Maps.newConcurrentMap();
context.put("counters", counters);
}
tables = context.get("tables");
if (tables == null) {
tables = Maps.newConcurrentMap();
......@@ -83,6 +90,13 @@ public class DefaultDatabaseState implements DatabaseState<String, byte[]> {
}
@Override
public Map<String, Long> counters() {
Map<String, Long> counterMap = Maps.newHashMap();
counters.forEach((k, v) -> counterMap.put(k, v.get()));
return counterMap;
}
@Override
public int size(String tableName) {
return getTableMap(tableName).size();
}
......@@ -212,6 +226,16 @@ public class DefaultDatabaseState implements DatabaseState<String, byte[]> {
}
@Override
public Long nextValue(String counterName) {
return getCounter(counterName).incrementAndGet();
}
@Override
public Long currentValue(String counterName) {
return getCounter(counterName).get();
}
@Override
public boolean prepareAndCommit(Transaction transaction) {
if (prepare(transaction)) {
return commit(transaction);
......@@ -255,6 +279,10 @@ public class DefaultDatabaseState implements DatabaseState<String, byte[]> {
return locks.computeIfAbsent(tableName, name -> Maps.newConcurrentMap());
}
private AtomicLong getCounter(String counterName) {
return counters.computeIfAbsent(counterName, name -> new AtomicLong(0));
}
private boolean isUpdatePossible(DatabaseUpdate update) {
Versioned<byte[]> existingEntry = get(update.tableName(), update.key());
switch (update.type()) {
......
......@@ -103,18 +103,7 @@ public class DistributedLeadershipManager implements LeadershipService {
public void activate() {
lockMap = storageService.<String, NodeId>consistentMapBuilder()
.withName("onos-leader-locks")
.withSerializer(new Serializer() {
KryoNamespace kryo = new KryoNamespace.Builder().register(KryoNamespaces.API).build();
@Override
public <T> byte[] encode(T object) {
return kryo.serialize(object);
}
@Override
public <T> T decode(byte[] bytes) {
return kryo.deserialize(bytes);
}
})
.withSerializer(Serializer.using(new KryoNamespace.Builder().register(KryoNamespaces.API).build()))
.withPartitionsDisabled().build();
localNodeId = clusterService.getLocalNode().id();
......
......@@ -90,6 +90,21 @@ public class PartitionedDatabase implements Database {
}
@Override
public CompletableFuture<Map<String, Long>> counters() {
checkState(isOpen.get(), DB_NOT_OPEN);
Map<String, Long> counters = Maps.newConcurrentMap();
return CompletableFuture.allOf(partitions
.stream()
.map(db -> db.counters()
.thenApply(m -> {
counters.putAll(m);
return null;
}))
.toArray(CompletableFuture[]::new))
.thenApply(v -> counters);
}
@Override
public CompletableFuture<Integer> size(String tableName) {
checkState(isOpen.get(), DB_NOT_OPEN);
AtomicInteger totalSize = new AtomicInteger(0);
......@@ -219,6 +234,18 @@ public class PartitionedDatabase implements Database {
}
@Override
public CompletableFuture<Long> nextValue(String counterName) {
checkState(isOpen.get(), DB_NOT_OPEN);
return partitioner.getPartition(counterName, counterName).nextValue(counterName);
}
@Override
public CompletableFuture<Long> currentValue(String counterName) {
checkState(isOpen.get(), DB_NOT_OPEN);
return partitioner.getPartition(counterName, counterName).currentValue(counterName);
}
@Override
public CompletableFuture<Boolean> prepareAndCommit(Transaction transaction) {
Map<Database, Transaction> subTransactions = createSubTransactions(transaction);
if (subTransactions.isEmpty()) {
......
......@@ -36,34 +36,22 @@ import org.onosproject.store.service.Transaction.State;
*/
public class TransactionManager {
private static final KryoNamespace KRYO_NAMESPACE = KryoNamespace.newBuilder()
.register(KryoNamespaces.BASIC)
.nextId(KryoNamespace.FLOATING_ID)
.register(Versioned.class)
.register(DatabaseUpdate.class)
.register(DatabaseUpdate.Type.class)
.register(DefaultTransaction.class)
.register(Transaction.State.class)
.register(Pair.class)
.register(ImmutablePair.class)
.build();
private final Serializer serializer = Serializer.using(KRYO_NAMESPACE);
private final Database database;
private final AsyncConsistentMap<Long, Transaction> transactions;
private final Serializer serializer = new Serializer() {
private KryoNamespace kryo = KryoNamespace.newBuilder()
.register(KryoNamespaces.BASIC)
.nextId(KryoNamespace.FLOATING_ID)
.register(Versioned.class)
.register(DatabaseUpdate.class)
.register(DatabaseUpdate.Type.class)
.register(DefaultTransaction.class)
.register(Transaction.State.class)
.register(Pair.class)
.register(ImmutablePair.class)
.build();
@Override
public <T> byte[] encode(T object) {
return kryo.serialize(object);
}
@Override
public <T> T decode(byte[] bytes) {
return kryo.deserialize(bytes);
}
};
/**
* Constructs a new TransactionManager for the specified database instance.
*
......