홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Madan Jampani
Committed by Gerrit Code Review
3780d4b0 1 parent 284fb69b

ONOS-4218: Fixes for resource store transaction failures 

Change-Id: Ie48bb04d7daf6ed7b63c33a3c3c2703496179aa6
Inline Side-by-side
Showing 8 changed files with 99 additions and 57 deletions
...@@ -21,6 +21,8 @@ import static com.google.common.base.Preconditions.checkState; ...@@ -21,6 +21,8 @@ import static com.google.common.base.Preconditions.checkState;
21 21
22 import java.util.function.Function; 22 import java.util.function.Function;
23 23
24 +import org.onlab.util.ByteArraySizeHashPrinter;
25 +
24 import com.google.common.base.MoreObjects; 26 import com.google.common.base.MoreObjects;
25 27
26 /** 28 /**
...@@ -153,7 +155,7 @@ public final class MapUpdate<K, V> { ...@@ -153,7 +155,7 @@ public final class MapUpdate<K, V> {
153 .add("mapName", mapName) 155 .add("mapName", mapName)
154 .add("type", type) 156 .add("type", type)
155 .add("key", key) 157 .add("key", key)
156 - .add("value", value) 158 + .add("value", value instanceof byte[] ? new ByteArraySizeHashPrinter((byte[]) value) : value)
157 .add("currentValue", currentValue) 159 .add("currentValue", currentValue)
158 .add("currentVersion", currentVersion) 160 .add("currentVersion", currentVersion)
159 .toString(); 161 .toString();
......
1 +/*
2 + * Copyright 2016 Open Networking Laboratory
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +package org.onosproject.store.service;
17 +
18 +/**
19 + * Completion status of transaction.
20 + */
21 +public enum CommitStatus {
22 + /**
23 + * Indicates a successfully completed transaction with all the updates committed.
24 + */
25 + SUCCESS,
26 +
27 + /**
28 + * Indicates a aborted transaction i.e. no updates were committed.
29 + */
30 + FAILURE
31 +}
...\ No newline at end of file ...\ No newline at end of file
...@@ -16,6 +16,8 @@ ...@@ -16,6 +16,8 @@
16 16
17 package org.onosproject.store.service; 17 package org.onosproject.store.service;
18 18
19 +import java.util.concurrent.CompletableFuture;
20 +
19 import org.onosproject.store.primitives.TransactionId; 21 import org.onosproject.store.primitives.TransactionId;
20 22
21 /** 23 /**
...@@ -63,9 +65,9 @@ public interface TransactionContext extends DistributedPrimitive { ...@@ -63,9 +65,9 @@ public interface TransactionContext extends DistributedPrimitive {
63 * Commits a transaction that was previously started thereby making its changes permanent 65 * Commits a transaction that was previously started thereby making its changes permanent
64 * and externally visible. 66 * and externally visible.
65 * 67 *
66 - * @return true if this transaction succeeded, otherwise false. 68 + * @return A future that will be completed when the operation completes
67 */ 69 */
68 - boolean commit(); 70 + CompletableFuture<CommitStatus> commit();
69 71
70 /** 72 /**
71 * Aborts any changes made in this transaction context and discarding all locally cached updates. 73 * Aborts any changes made in this transaction context and discarding all locally cached updates.
......
...@@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableList; ...@@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableList;
20 import com.google.common.collect.ImmutableSet; 20 import com.google.common.collect.ImmutableSet;
21 import com.google.common.collect.Maps; 21 import com.google.common.collect.Maps;
22 import com.google.common.collect.Sets; 22 import com.google.common.collect.Sets;
23 +
23 import org.apache.felix.scr.annotations.Activate; 24 import org.apache.felix.scr.annotations.Activate;
24 import org.apache.felix.scr.annotations.Component; 25 import org.apache.felix.scr.annotations.Component;
25 import org.apache.felix.scr.annotations.Reference; 26 import org.apache.felix.scr.annotations.Reference;
...@@ -41,6 +42,7 @@ import org.onosproject.net.resource.ResourceStoreDelegate; ...@@ -41,6 +42,7 @@ import org.onosproject.net.resource.ResourceStoreDelegate;
41 import org.onosproject.net.resource.Resources; 42 import org.onosproject.net.resource.Resources;
42 import org.onosproject.store.AbstractStore; 43 import org.onosproject.store.AbstractStore;
43 import org.onosproject.store.serializers.KryoNamespaces; 44 import org.onosproject.store.serializers.KryoNamespaces;
45 +import org.onosproject.store.service.CommitStatus;
44 import org.onosproject.store.service.ConsistentMap; 46 import org.onosproject.store.service.ConsistentMap;
45 import org.onosproject.store.service.ConsistentMapException; 47 import org.onosproject.store.service.ConsistentMapException;
46 import org.onosproject.store.service.Serializer; 48 import org.onosproject.store.service.Serializer;
...@@ -178,18 +180,18 @@ public class ConsistentResourceStore extends AbstractStore<ResourceEvent, Resour ...@@ -178,18 +180,18 @@ public class ConsistentResourceStore extends AbstractStore<ResourceEvent, Resour
178 } 180 }
179 } 181 }
180 182
181 - boolean success = tx.commit(); 183 + return tx.commit().whenComplete((status, error) -> {
182 - if (success) { 184 + if (status == CommitStatus.SUCCESS) {
183 - log.trace("Transaction commit succeeded on registration: resources={}", resources); 185 + log.trace("Transaction commit succeeded on registration: resources={}", resources);
184 - List<ResourceEvent> events = resources.stream() 186 + List<ResourceEvent> events = resources.stream()
185 - .filter(x -> x.parent().isPresent()) 187 + .filter(x -> x.parent().isPresent())
186 - .map(x -> new ResourceEvent(RESOURCE_ADDED, x)) 188 + .map(x -> new ResourceEvent(RESOURCE_ADDED, x))
187 - .collect(Collectors.toList()); 189 + .collect(Collectors.toList());
188 - notifyDelegate(events); 190 + notifyDelegate(events);
189 - } else { 191 + } else {
190 - log.debug("Transaction commit failed on registration: resources={}", resources); 192 + log.warn("Transaction commit failed on registration", error);
191 - } 193 + }
192 - return success; 194 + }).join() == CommitStatus.SUCCESS;
193 } 195 }
194 196
195 @Override 197 @Override
...@@ -252,17 +254,17 @@ public class ConsistentResourceStore extends AbstractStore<ResourceEvent, Resour ...@@ -252,17 +254,17 @@ public class ConsistentResourceStore extends AbstractStore<ResourceEvent, Resour
252 } 254 }
253 } 255 }
254 256
255 - boolean success = tx.commit(); 257 + return tx.commit().whenComplete((status, error) -> {
256 - if (success) { 258 + if (status == CommitStatus.SUCCESS) {
257 - List<ResourceEvent> events = resources.stream() 259 + List<ResourceEvent> events = resources.stream()
258 - .filter(x -> x.parent().isPresent()) 260 + .filter(x -> x.parent().isPresent())
259 - .map(x -> new ResourceEvent(RESOURCE_REMOVED, x)) 261 + .map(x -> new ResourceEvent(RESOURCE_REMOVED, x))
260 - .collect(Collectors.toList()); 262 + .collect(Collectors.toList());
261 - notifyDelegate(events); 263 + notifyDelegate(events);
262 - } else { 264 + } else {
263 - log.warn("Failed to unregister {}: Commit failed.", ids); 265 + log.warn("Failed to unregister {}: Commit failed.", ids, error);
264 - } 266 + }
265 - return success; 267 + }).join() == CommitStatus.SUCCESS;
266 } 268 }
267 269
268 @Override 270 @Override
...@@ -308,7 +310,7 @@ public class ConsistentResourceStore extends AbstractStore<ResourceEvent, Resour ...@@ -308,7 +310,7 @@ public class ConsistentResourceStore extends AbstractStore<ResourceEvent, Resour
308 } 310 }
309 } 311 }
310 312
311 - return tx.commit(); 313 + return tx.commit().join() == CommitStatus.SUCCESS;
312 } 314 }
313 315
314 @Override 316 @Override
...@@ -348,7 +350,7 @@ public class ConsistentResourceStore extends AbstractStore<ResourceEvent, Resour ...@@ -348,7 +350,7 @@ public class ConsistentResourceStore extends AbstractStore<ResourceEvent, Resour
348 } 350 }
349 } 351 }
350 352
351 - return tx.commit(); 353 + return tx.commit().join() == CommitStatus.SUCCESS;
352 } 354 }
353 355
354 // computational complexity: O(1) if the resource is discrete type. 356 // computational complexity: O(1) if the resource is discrete type.
......
...@@ -27,6 +27,7 @@ import static com.google.common.base.Preconditions.*; ...@@ -27,6 +27,7 @@ import static com.google.common.base.Preconditions.*;
27 import org.onosproject.store.primitives.MapUpdate; 27 import org.onosproject.store.primitives.MapUpdate;
28 import org.onosproject.store.primitives.TransactionId; 28 import org.onosproject.store.primitives.TransactionId;
29 import org.onosproject.store.primitives.resources.impl.CommitResult; 29 import org.onosproject.store.primitives.resources.impl.CommitResult;
30 +import org.onosproject.store.service.CommitStatus;
30 import org.onosproject.store.service.ConsistentMapBuilder; 31 import org.onosproject.store.service.ConsistentMapBuilder;
31 import org.onosproject.store.service.Serializer; 32 import org.onosproject.store.service.Serializer;
32 import org.onosproject.store.service.TransactionContext; 33 import org.onosproject.store.service.TransactionContext;
...@@ -96,22 +97,23 @@ public class DefaultTransactionContext implements TransactionContext { ...@@ -96,22 +97,23 @@ public class DefaultTransactionContext implements TransactionContext {
96 97
97 @SuppressWarnings("unchecked") 98 @SuppressWarnings("unchecked")
98 @Override 99 @Override
99 - public boolean commit() { 100 + public CompletableFuture<CommitStatus> commit() {
100 // TODO: rework commit implementation to be more intuitive 101 // TODO: rework commit implementation to be more intuitive
101 checkState(isOpen, TX_NOT_OPEN_ERROR); 102 checkState(isOpen, TX_NOT_OPEN_ERROR);
102 - CommitResult result = null; 103 + CommitStatus status;
103 try { 104 try {
104 List<MapUpdate<String, byte[]>> updates = Lists.newLinkedList(); 105 List<MapUpdate<String, byte[]>> updates = Lists.newLinkedList();
105 txMaps.values().forEach(m -> updates.addAll(m.toMapUpdates())); 106 txMaps.values().forEach(m -> updates.addAll(m.toMapUpdates()));
106 Transaction transaction = new Transaction(transactionId, updates); 107 Transaction transaction = new Transaction(transactionId, updates);
107 - result = Futures.getUnchecked(transactionCommitter.apply(transaction)); 108 + status = Futures.getUnchecked(transactionCommitter.apply(transaction)) == CommitResult.OK
108 - return result == CommitResult.OK; 109 + ? CommitStatus.SUCCESS : CommitStatus.FAILURE;
109 } catch (Exception e) { 110 } catch (Exception e) {
110 abort(); 111 abort();
111 - return false; 112 + status = CommitStatus.FAILURE;
112 } finally { 113 } finally {
113 isOpen = false; 114 isOpen = false;
114 } 115 }
116 + return CompletableFuture.completedFuture(status);
115 } 117 }
116 118
117 @Override 119 @Override
......
...@@ -16,13 +16,16 @@ ...@@ -16,13 +16,16 @@
16 package org.onosproject.store.primitives.impl; 16 package org.onosproject.store.primitives.impl;
17 17
18 import java.util.Set; 18 import java.util.Set;
19 +import java.util.concurrent.CompletableFuture;
19 import java.util.concurrent.atomic.AtomicBoolean; 20 import java.util.concurrent.atomic.AtomicBoolean;
20 21
21 import org.onosproject.store.primitives.DistributedPrimitiveCreator; 22 import org.onosproject.store.primitives.DistributedPrimitiveCreator;
22 import org.onosproject.store.primitives.TransactionId; 23 import org.onosproject.store.primitives.TransactionId;
24 +import org.onosproject.store.service.CommitStatus;
23 import org.onosproject.store.service.Serializer; 25 import org.onosproject.store.service.Serializer;
24 import org.onosproject.store.service.TransactionContext; 26 import org.onosproject.store.service.TransactionContext;
25 import org.onosproject.store.service.TransactionalMap; 27 import org.onosproject.store.service.TransactionalMap;
28 +import org.onosproject.utils.MeteringAgent;
26 29
27 import com.google.common.collect.Sets; 30 import com.google.common.collect.Sets;
28 31
...@@ -36,6 +39,7 @@ public class NewDefaultTransactionContext implements TransactionContext { ...@@ -36,6 +39,7 @@ public class NewDefaultTransactionContext implements TransactionContext {
36 private final TransactionId transactionId; 39 private final TransactionId transactionId;
37 private final TransactionCoordinator transactionCoordinator; 40 private final TransactionCoordinator transactionCoordinator;
38 private final Set<TransactionParticipant> txParticipants = Sets.newConcurrentHashSet(); 41 private final Set<TransactionParticipant> txParticipants = Sets.newConcurrentHashSet();
42 + private final MeteringAgent monitor;
39 43
40 public NewDefaultTransactionContext(TransactionId transactionId, 44 public NewDefaultTransactionContext(TransactionId transactionId,
41 DistributedPrimitiveCreator creator, 45 DistributedPrimitiveCreator creator,
...@@ -43,6 +47,7 @@ public class NewDefaultTransactionContext implements TransactionContext { ...@@ -43,6 +47,7 @@ public class NewDefaultTransactionContext implements TransactionContext {
43 this.transactionId = transactionId; 47 this.transactionId = transactionId;
44 this.creator = creator; 48 this.creator = creator;
45 this.transactionCoordinator = transactionCoordinator; 49 this.transactionCoordinator = transactionCoordinator;
50 + this.monitor = new MeteringAgent("transactionContext", "*", true);
46 } 51 }
47 52
48 @Override 53 @Override
...@@ -68,9 +73,10 @@ public class NewDefaultTransactionContext implements TransactionContext { ...@@ -68,9 +73,10 @@ public class NewDefaultTransactionContext implements TransactionContext {
68 } 73 }
69 74
70 @Override 75 @Override
71 - public boolean commit() { 76 + public CompletableFuture<CommitStatus> commit() {
72 - transactionCoordinator.commit(transactionId, txParticipants).getNow(null); 77 + final MeteringAgent.Context timer = monitor.startTimer("commit");
73 - return true; 78 + return transactionCoordinator.commit(transactionId, txParticipants)
79 + .whenComplete((r, e) -> timer.stop(e));
74 } 80 }
75 81
76 @Override 82 @Override
......
...@@ -22,6 +22,7 @@ import java.util.stream.Collectors; ...@@ -22,6 +22,7 @@ import java.util.stream.Collectors;
22 import org.onlab.util.Tools; 22 import org.onlab.util.Tools;
23 import org.onosproject.store.primitives.TransactionId; 23 import org.onosproject.store.primitives.TransactionId;
24 import org.onosproject.store.service.AsyncConsistentMap; 24 import org.onosproject.store.service.AsyncConsistentMap;
25 +import org.onosproject.store.service.CommitStatus;
25 26
26 /** 27 /**
27 * Coordinator for a two-phase commit protocol. 28 * Coordinator for a two-phase commit protocol.
...@@ -37,45 +38,47 @@ public class TransactionCoordinator { ...@@ -37,45 +38,47 @@ public class TransactionCoordinator {
37 /** 38 /**
38 * Commits a transaction. 39 * Commits a transaction.
39 * 40 *
40 - * @param transactionId transaction 41 + * @param transactionId transaction identifier
41 * @param transactionParticipants set of transaction participants 42 * @param transactionParticipants set of transaction participants
42 * @return future for commit result 43 * @return future for commit result
43 */ 44 */
44 - CompletableFuture<Void> commit(TransactionId transactionId, Set<TransactionParticipant> transactionParticipants) { 45 + CompletableFuture<CommitStatus> commit(TransactionId transactionId,
46 + Set<TransactionParticipant> transactionParticipants) {
45 if (!transactionParticipants.stream().anyMatch(t -> t.hasPendingUpdates())) { 47 if (!transactionParticipants.stream().anyMatch(t -> t.hasPendingUpdates())) {
46 - return CompletableFuture.completedFuture(null); 48 + return CompletableFuture.completedFuture(CommitStatus.SUCCESS);
47 } 49 }
48 50
49 - return transactions.put(transactionId, Transaction.State.PREPARING) 51 + CompletableFuture<CommitStatus> status = transactions.put(transactionId, Transaction.State.PREPARING)
50 .thenCompose(v -> this.doPrepare(transactionParticipants)) 52 .thenCompose(v -> this.doPrepare(transactionParticipants))
51 .thenCompose(result -> result 53 .thenCompose(result -> result
52 ? transactions.put(transactionId, Transaction.State.COMMITTING) 54 ? transactions.put(transactionId, Transaction.State.COMMITTING)
53 .thenCompose(v -> doCommit(transactionParticipants)) 55 .thenCompose(v -> doCommit(transactionParticipants))
54 - .thenApply(v -> null) 56 + .thenApply(v -> CommitStatus.SUCCESS)
55 : transactions.put(transactionId, Transaction.State.ROLLINGBACK) 57 : transactions.put(transactionId, Transaction.State.ROLLINGBACK)
56 .thenCompose(v -> doRollback(transactionParticipants)) 58 .thenCompose(v -> doRollback(transactionParticipants))
57 - .thenApply(v -> null)) 59 + .thenApply(v -> CommitStatus.FAILURE));
58 - .thenCompose(v -> transactions.remove(transactionId)) 60 + return status.thenCompose(v -> transactions.remove(transactionId).thenApply(u -> v));
59 - .thenApply(v -> null);
60 } 61 }
61 62
62 private CompletableFuture<Boolean> doPrepare(Set<TransactionParticipant> transactionParticipants) { 63 private CompletableFuture<Boolean> doPrepare(Set<TransactionParticipant> transactionParticipants) {
63 - return Tools.allOf(transactionParticipants 64 + return Tools.allOf(transactionParticipants.stream()
64 - .stream() 65 + .filter(TransactionParticipant::hasPendingUpdates)
65 - .map(TransactionParticipant::prepare) 66 + .map(TransactionParticipant::prepare)
66 - .collect(Collectors.toList())) 67 + .collect(Collectors.toList()))
67 .thenApply(list -> list.stream().reduce(Boolean::logicalAnd).orElse(true)); 68 .thenApply(list -> list.stream().reduce(Boolean::logicalAnd).orElse(true));
68 } 69 }
69 70
70 private CompletableFuture<Void> doCommit(Set<TransactionParticipant> transactionParticipants) { 71 private CompletableFuture<Void> doCommit(Set<TransactionParticipant> transactionParticipants) {
71 return CompletableFuture.allOf(transactionParticipants.stream() 72 return CompletableFuture.allOf(transactionParticipants.stream()
72 - .map(p -> p.commit()) 73 + .filter(TransactionParticipant::hasPendingUpdates)
74 + .map(TransactionParticipant::commit)
73 .toArray(CompletableFuture[]::new)); 75 .toArray(CompletableFuture[]::new));
74 } 76 }
75 77
76 private CompletableFuture<Void> doRollback(Set<TransactionParticipant> transactionParticipants) { 78 private CompletableFuture<Void> doRollback(Set<TransactionParticipant> transactionParticipants) {
77 return CompletableFuture.allOf(transactionParticipants.stream() 79 return CompletableFuture.allOf(transactionParticipants.stream()
78 - .map(p -> p.rollback()) 80 + .filter(TransactionParticipant::hasPendingUpdates)
81 + .map(TransactionParticipant::rollback)
79 .toArray(CompletableFuture[]::new)); 82 .toArray(CompletableFuture[]::new));
80 } 83 }
81 } 84 }
...\ No newline at end of file ...\ No newline at end of file
......
...@@ -87,13 +87,7 @@ public class AtomixConsistentMap extends AbstractResource<AtomixConsistentMap> ...@@ -87,13 +87,7 @@ public class AtomixConsistentMap extends AbstractResource<AtomixConsistentMap>
87 } 87 }
88 88
89 private void handleEvent(List<MapEvent<String, byte[]>> events) { 89 private void handleEvent(List<MapEvent<String, byte[]>> events) {
90 - events.forEach(event -> mapEventListeners.forEach(listener -> { 90 + events.forEach(event -> mapEventListeners.forEach(listener -> listener.event(event)));
91 - try {
92 - listener.event(event);
93 - } catch (Exception e) {
94 - log.warn("Error processing map event", e);
95 - }
96 - }));
97 } 91 }
98 92
99 @Override 93 @Override
......