홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Yuta HIGUCHI
Committed by Yuta Higuchi
4490a73a 1 parent a8016e71

DistributedIntentStore: CopyCat version of Distributed intent store 

- old DistributedIntentStore renamed to Hazelcast~ and is by default disabled

Change-Id: I386eaf6c136f8a2fbebb4268d20b1395249e77ea
Inline Side-by-side
Showing 5 changed files with 300 additions and 51 deletions
...@@ -34,6 +34,7 @@ import static com.google.common.base.Preconditions.checkNotNull; ...@@ -34,6 +34,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
34 import static com.google.common.base.Preconditions.checkState; 34 import static com.google.common.base.Preconditions.checkState;
35 import static org.slf4j.LoggerFactory.getLogger; 35 import static org.slf4j.LoggerFactory.getLogger;
36 36
37 +// FIXME This is not distributed yet.
37 @Component(immediate = true) 38 @Component(immediate = true)
38 @Service 39 @Service
39 public class DistributedIntentBatchQueue implements IntentBatchService { 40 public class DistributedIntentBatchQueue implements IntentBatchService {
......
...@@ -16,15 +16,12 @@ ...@@ -16,15 +16,12 @@
16 package org.onlab.onos.store.intent.impl; 16 package org.onlab.onos.store.intent.impl;
17 17
18 import com.google.common.collect.ImmutableSet; 18 import com.google.common.collect.ImmutableSet;
19 -import com.hazelcast.core.EntryAdapter;
20 -import com.hazelcast.core.EntryEvent;
21 -import com.hazelcast.core.EntryListener;
22 -import com.hazelcast.core.IMap;
23 -import com.hazelcast.core.Member;
24 19
25 import org.apache.felix.scr.annotations.Activate; 20 import org.apache.felix.scr.annotations.Activate;
26 import org.apache.felix.scr.annotations.Component; 21 import org.apache.felix.scr.annotations.Component;
27 import org.apache.felix.scr.annotations.Deactivate; 22 import org.apache.felix.scr.annotations.Deactivate;
23 +import org.apache.felix.scr.annotations.Reference;
24 +import org.apache.felix.scr.annotations.ReferenceCardinality;
28 import org.apache.felix.scr.annotations.Service; 25 import org.apache.felix.scr.annotations.Service;
29 import org.onlab.onos.net.intent.Intent; 26 import org.onlab.onos.net.intent.Intent;
30 import org.onlab.onos.net.intent.IntentEvent; 27 import org.onlab.onos.net.intent.IntentEvent;
...@@ -32,10 +29,13 @@ import org.onlab.onos.net.intent.IntentId; ...@@ -32,10 +29,13 @@ import org.onlab.onos.net.intent.IntentId;
32 import org.onlab.onos.net.intent.IntentState; 29 import org.onlab.onos.net.intent.IntentState;
33 import org.onlab.onos.net.intent.IntentStore; 30 import org.onlab.onos.net.intent.IntentStore;
34 import org.onlab.onos.net.intent.IntentStoreDelegate; 31 import org.onlab.onos.net.intent.IntentStoreDelegate;
35 -import org.onlab.onos.store.hz.AbstractHazelcastStore; 32 +import org.onlab.onos.store.AbstractStore;
36 -import org.onlab.onos.store.hz.SMap;
37 import org.onlab.onos.store.serializers.KryoNamespaces; 33 import org.onlab.onos.store.serializers.KryoNamespaces;
38 import org.onlab.onos.store.serializers.KryoSerializer; 34 import org.onlab.onos.store.serializers.KryoSerializer;
35 +import org.onlab.onos.store.serializers.StoreSerializer;
36 +import org.onlab.onos.store.service.DatabaseAdminService;
37 +import org.onlab.onos.store.service.DatabaseService;
38 +import org.onlab.onos.store.service.impl.CMap;
39 import org.onlab.util.KryoNamespace; 39 import org.onlab.util.KryoNamespace;
40 import org.slf4j.Logger; 40 import org.slf4j.Logger;
41 41
...@@ -52,7 +52,7 @@ import static org.slf4j.LoggerFactory.getLogger; ...@@ -52,7 +52,7 @@ import static org.slf4j.LoggerFactory.getLogger;
52 @Component(immediate = true) 52 @Component(immediate = true)
53 @Service 53 @Service
54 public class DistributedIntentStore 54 public class DistributedIntentStore
55 - extends AbstractHazelcastStore<IntentEvent, IntentStoreDelegate> 55 + extends AbstractStore<IntentEvent, IntentStoreDelegate>
56 implements IntentStore { 56 implements IntentStore {
57 57
58 /** Valid parking state, which can transition to INSTALLED. */ 58 /** Valid parking state, which can transition to INSTALLED. */
...@@ -64,22 +64,29 @@ public class DistributedIntentStore ...@@ -64,22 +64,29 @@ public class DistributedIntentStore
64 private final Logger log = getLogger(getClass()); 64 private final Logger log = getLogger(getClass());
65 65
66 // Assumption: IntentId will not have synonyms 66 // Assumption: IntentId will not have synonyms
67 - private SMap<IntentId, Intent> intents; 67 + private CMap<IntentId, Intent> intents;
68 - private SMap<IntentId, IntentState> states; 68 + private CMap<IntentId, IntentState> states;
69 69
70 + // TODO left behind transient state issue: ONOS-103
70 // Map to store instance local intermediate state transition 71 // Map to store instance local intermediate state transition
71 private transient Map<IntentId, IntentState> transientStates = new ConcurrentHashMap<>(); 72 private transient Map<IntentId, IntentState> transientStates = new ConcurrentHashMap<>();
72 73
73 - private SMap<IntentId, List<Intent>> installable; 74 + private CMap<IntentId, List<Intent>> installable;
75 +
76 + private StoreSerializer serializer;
77 +
78 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
79 + protected DatabaseAdminService dbAdminService;
80 +
81 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
82 + protected DatabaseService dbService;
74 83
75 - @Override
76 @Activate 84 @Activate
77 public void activate() { 85 public void activate() {
78 // FIXME: We need a way to add serializer for intents which has been plugged-in. 86 // FIXME: We need a way to add serializer for intents which has been plugged-in.
79 // As a short term workaround, relax Kryo config to 87 // As a short term workaround, relax Kryo config to
80 // registrationRequired=false 88 // registrationRequired=false
81 - super.activate(); 89 + serializer = new KryoSerializer() {
82 - super.serializer = new KryoSerializer() {
83 90
84 @Override 91 @Override
85 protected void setupKryoPool() { 92 protected void setupKryoPool() {
...@@ -89,24 +96,15 @@ public class DistributedIntentStore ...@@ -89,24 +96,15 @@ public class DistributedIntentStore
89 .build() 96 .build()
90 .populate(1); 97 .populate(1);
91 } 98 }
92 -
93 }; 99 };
94 100
95 - // TODO: enable near cache, allow read from backup for this IMap 101 + intents = new CMap<>(dbAdminService, dbService, "intents", serializer);
96 - IMap<byte[], byte[]> rawIntents = super.theInstance.getMap("intents");
97 - intents = new SMap<>(rawIntents , super.serializer);
98 102
99 - // TODO: disable near cache, disable read from backup for this IMap 103 + states = new CMap<>(dbAdminService, dbService, "intent-states", serializer);
100 - IMap<byte[], byte[]> rawStates = super.theInstance.getMap("intent-states");
101 - states = new SMap<>(rawStates , super.serializer);
102 - EntryListener<IntentId, IntentState> listener = new RemoteIntentStateListener();
103 - states.addEntryListener(listener , false);
104 104
105 transientStates.clear(); 105 transientStates.clear();
106 106
107 - // TODO: disable near cache, disable read from backup for this IMap 107 + installable = new CMap<>(dbAdminService, dbService, "installable-intents", serializer);
108 - IMap<byte[], byte[]> rawInstallables = super.theInstance.getMap("installable-intents");
109 - installable = new SMap<>(rawInstallables , super.serializer);
110 108
111 log.info("Started"); 109 log.info("Started");
112 } 110 }
...@@ -118,8 +116,8 @@ public class DistributedIntentStore ...@@ -118,8 +116,8 @@ public class DistributedIntentStore
118 116
119 @Override 117 @Override
120 public IntentEvent createIntent(Intent intent) { 118 public IntentEvent createIntent(Intent intent) {
121 - Intent existing = intents.putIfAbsent(intent.id(), intent); 119 + boolean absent = intents.putIfAbsent(intent.id(), intent);
122 - if (existing != null) { 120 + if (!absent) {
123 // duplicate, ignore 121 // duplicate, ignore
124 return null; 122 return null;
125 } else { 123 } else {
...@@ -173,34 +171,47 @@ public class DistributedIntentStore ...@@ -173,34 +171,47 @@ public class DistributedIntentStore
173 IntentEvent.Type type = null; 171 IntentEvent.Type type = null;
174 final IntentState prevParking; 172 final IntentState prevParking;
175 boolean transientStateChangeOnly = false; 173 boolean transientStateChangeOnly = false;
174 + boolean updated;
176 175
177 // parking state transition 176 // parking state transition
178 switch (state) { 177 switch (state) {
179 case SUBMITTED: 178 case SUBMITTED:
180 - prevParking = states.putIfAbsent(id, SUBMITTED); 179 + prevParking = states.get(id);
181 verify(prevParking == null, 180 verify(prevParking == null,
182 "Illegal state transition attempted from %s to SUBMITTED", 181 "Illegal state transition attempted from %s to SUBMITTED",
183 prevParking); 182 prevParking);
183 + updated = states.putIfAbsent(id, SUBMITTED);
184 + verify(updated, "Conditional replace %s => %s failed", prevParking, SUBMITTED);
184 type = IntentEvent.Type.SUBMITTED; 185 type = IntentEvent.Type.SUBMITTED;
185 break; 186 break;
187 +
186 case INSTALLED: 188 case INSTALLED:
187 - prevParking = states.replace(id, INSTALLED); 189 + prevParking = states.get(id);
188 verify(PRE_INSTALLED.contains(prevParking), 190 verify(PRE_INSTALLED.contains(prevParking),
189 "Illegal state transition attempted from %s to INSTALLED", 191 "Illegal state transition attempted from %s to INSTALLED",
190 prevParking); 192 prevParking);
193 + updated = states.replace(id, prevParking, INSTALLED);
194 + verify(updated, "Conditional replace %s => %s failed", prevParking, INSTALLED);
191 type = IntentEvent.Type.INSTALLED; 195 type = IntentEvent.Type.INSTALLED;
192 break; 196 break;
197 +
193 case FAILED: 198 case FAILED:
194 - prevParking = states.replace(id, FAILED); 199 + prevParking = states.get(id);
200 + updated = states.replace(id, prevParking, FAILED);
201 + verify(updated, "Conditional replace %s => %s failed", prevParking, FAILED);
195 type = IntentEvent.Type.FAILED; 202 type = IntentEvent.Type.FAILED;
196 break; 203 break;
204 +
197 case WITHDRAWN: 205 case WITHDRAWN:
198 - prevParking = states.replace(id, WITHDRAWN); 206 + prevParking = states.get(id);
199 verify(PRE_WITHDRAWN.contains(prevParking), 207 verify(PRE_WITHDRAWN.contains(prevParking),
200 "Illegal state transition attempted from %s to WITHDRAWN", 208 "Illegal state transition attempted from %s to WITHDRAWN",
201 prevParking); 209 prevParking);
210 + updated = states.replace(id, prevParking, WITHDRAWN);
211 + verify(updated, "Conditional replace %s => %s failed", prevParking, WITHDRAWN);
202 type = IntentEvent.Type.WITHDRAWN; 212 type = IntentEvent.Type.WITHDRAWN;
203 break; 213 break;
214 +
204 default: 215 default:
205 transientStateChangeOnly = true; 216 transientStateChangeOnly = true;
206 prevParking = null; 217 prevParking = null;
...@@ -233,22 +244,4 @@ public class DistributedIntentStore ...@@ -233,22 +244,4 @@ public class DistributedIntentStore
233 public void removeInstalledIntents(IntentId intentId) { 244 public void removeInstalledIntents(IntentId intentId) {
234 installable.remove(intentId); 245 installable.remove(intentId);
235 } 246 }
236 -
237 - public final class RemoteIntentStateListener extends EntryAdapter<IntentId, IntentState> {
238 -
239 - @Override
240 - public void onEntryEvent(EntryEvent<IntentId, IntentState> event) {
241 - final Member myself = theInstance.getCluster().getLocalMember();
242 - if (!myself.equals(event.getMember())) {
243 - // When Intent state was modified by remote node,
244 - // clear local transient state.
245 - final IntentId intentId = event.getKey();
246 - IntentState oldState = transientStates.remove(intentId);
247 - if (oldState != null) {
248 - log.debug("{} state updated remotely, removing transient state {}",
249 - intentId, oldState);
250 - }
251 - }
252 - }
253 - }
254 } 247 }
......
1 +/*
2 + * Copyright 2014 Open Networking Laboratory
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +package org.onlab.onos.store.intent.impl;
17 +
18 +import com.google.common.collect.ImmutableSet;
19 +import com.hazelcast.core.EntryAdapter;
20 +import com.hazelcast.core.EntryEvent;
21 +import com.hazelcast.core.EntryListener;
22 +import com.hazelcast.core.IMap;
23 +import com.hazelcast.core.Member;
24 +
25 +import org.apache.felix.scr.annotations.Activate;
26 +import org.apache.felix.scr.annotations.Component;
27 +import org.apache.felix.scr.annotations.Deactivate;
28 +import org.apache.felix.scr.annotations.Service;
29 +import org.onlab.onos.net.intent.Intent;
30 +import org.onlab.onos.net.intent.IntentEvent;
31 +import org.onlab.onos.net.intent.IntentId;
32 +import org.onlab.onos.net.intent.IntentState;
33 +import org.onlab.onos.net.intent.IntentStore;
34 +import org.onlab.onos.net.intent.IntentStoreDelegate;
35 +import org.onlab.onos.store.hz.AbstractHazelcastStore;
36 +import org.onlab.onos.store.hz.SMap;
37 +import org.onlab.onos.store.serializers.KryoNamespaces;
38 +import org.onlab.onos.store.serializers.KryoSerializer;
39 +import org.onlab.util.KryoNamespace;
40 +import org.slf4j.Logger;
41 +
42 +import java.util.EnumSet;
43 +import java.util.List;
44 +import java.util.Map;
45 +import java.util.Set;
46 +import java.util.concurrent.ConcurrentHashMap;
47 +
48 +import static com.google.common.base.Verify.verify;
49 +import static org.onlab.onos.net.intent.IntentState.*;
50 +import static org.slf4j.LoggerFactory.getLogger;
51 +
52 +@Component(immediate = false, enabled = false)
53 +@Service
54 +public class HazelcastIntentStore
55 + extends AbstractHazelcastStore<IntentEvent, IntentStoreDelegate>
56 + implements IntentStore {
57 +
58 + /** Valid parking state, which can transition to INSTALLED. */
59 + private static final Set<IntentState> PRE_INSTALLED = EnumSet.of(SUBMITTED, FAILED);
60 +
61 + /** Valid parking state, which can transition to WITHDRAWN. */
62 + private static final Set<IntentState> PRE_WITHDRAWN = EnumSet.of(INSTALLED, FAILED);
63 +
64 + private final Logger log = getLogger(getClass());
65 +
66 + // Assumption: IntentId will not have synonyms
67 + private SMap<IntentId, Intent> intents;
68 + private SMap<IntentId, IntentState> states;
69 +
70 + // Map to store instance local intermediate state transition
71 + private transient Map<IntentId, IntentState> transientStates = new ConcurrentHashMap<>();
72 +
73 + private SMap<IntentId, List<Intent>> installable;
74 +
75 + @Override
76 + @Activate
77 + public void activate() {
78 + // FIXME: We need a way to add serializer for intents which has been plugged-in.
79 + // As a short term workaround, relax Kryo config to
80 + // registrationRequired=false
81 + super.activate();
82 + super.serializer = new KryoSerializer() {
83 +
84 + @Override
85 + protected void setupKryoPool() {
86 + serializerPool = KryoNamespace.newBuilder()
87 + .setRegistrationRequired(false)
88 + .register(KryoNamespaces.API)
89 + .build()
90 + .populate(1);
91 + }
92 +
93 + };
94 +
95 + // TODO: enable near cache, allow read from backup for this IMap
96 + IMap<byte[], byte[]> rawIntents = super.theInstance.getMap("intents");
97 + intents = new SMap<>(rawIntents , super.serializer);
98 +
99 + // TODO: disable near cache, disable read from backup for this IMap
100 + IMap<byte[], byte[]> rawStates = super.theInstance.getMap("intent-states");
101 + states = new SMap<>(rawStates , super.serializer);
102 + EntryListener<IntentId, IntentState> listener = new RemoteIntentStateListener();
103 + states.addEntryListener(listener , false);
104 +
105 + transientStates.clear();
106 +
107 + // TODO: disable near cache, disable read from backup for this IMap
108 + IMap<byte[], byte[]> rawInstallables = super.theInstance.getMap("installable-intents");
109 + installable = new SMap<>(rawInstallables , super.serializer);
110 +
111 + log.info("Started");
112 + }
113 +
114 + @Deactivate
115 + public void deactivate() {
116 + log.info("Stopped");
117 + }
118 +
119 + @Override
120 + public IntentEvent createIntent(Intent intent) {
121 + Intent existing = intents.putIfAbsent(intent.id(), intent);
122 + if (existing != null) {
123 + // duplicate, ignore
124 + return null;
125 + } else {
126 + return this.setState(intent, IntentState.SUBMITTED);
127 + }
128 + }
129 +
130 + @Override
131 + public IntentEvent removeIntent(IntentId intentId) {
132 + Intent intent = intents.remove(intentId);
133 + installable.remove(intentId);
134 + if (intent == null) {
135 + // was already removed
136 + return null;
137 + }
138 + IntentEvent event = this.setState(intent, WITHDRAWN);
139 + states.remove(intentId);
140 + transientStates.remove(intentId);
141 + // TODO: Should we callremoveInstalledIntents if this Intent was
142 + return event;
143 + }
144 +
145 + @Override
146 + public long getIntentCount() {
147 + return intents.size();
148 + }
149 +
150 + @Override
151 + public Iterable<Intent> getIntents() {
152 + return ImmutableSet.copyOf(intents.values());
153 + }
154 +
155 + @Override
156 + public Intent getIntent(IntentId intentId) {
157 + return intents.get(intentId);
158 + }
159 +
160 + @Override
161 + public IntentState getIntentState(IntentId id) {
162 + final IntentState localState = transientStates.get(id);
163 + if (localState != null) {
164 + return localState;
165 + }
166 + return states.get(id);
167 + }
168 +
169 +
170 + @Override
171 + public IntentEvent setState(Intent intent, IntentState state) {
172 + final IntentId id = intent.id();
173 + IntentEvent.Type type = null;
174 + final IntentState prevParking;
175 + boolean transientStateChangeOnly = false;
176 +
177 + // parking state transition
178 + switch (state) {
179 + case SUBMITTED:
180 + prevParking = states.putIfAbsent(id, SUBMITTED);
181 + verify(prevParking == null,
182 + "Illegal state transition attempted from %s to SUBMITTED",
183 + prevParking);
184 + type = IntentEvent.Type.SUBMITTED;
185 + break;
186 + case INSTALLED:
187 + prevParking = states.replace(id, INSTALLED);
188 + verify(PRE_INSTALLED.contains(prevParking),
189 + "Illegal state transition attempted from %s to INSTALLED",
190 + prevParking);
191 + type = IntentEvent.Type.INSTALLED;
192 + break;
193 + case FAILED:
194 + prevParking = states.replace(id, FAILED);
195 + type = IntentEvent.Type.FAILED;
196 + break;
197 + case WITHDRAWN:
198 + prevParking = states.replace(id, WITHDRAWN);
199 + verify(PRE_WITHDRAWN.contains(prevParking),
200 + "Illegal state transition attempted from %s to WITHDRAWN",
201 + prevParking);
202 + type = IntentEvent.Type.WITHDRAWN;
203 + break;
204 + default:
205 + transientStateChangeOnly = true;
206 + prevParking = null;
207 + break;
208 + }
209 + if (!transientStateChangeOnly) {
210 + log.debug("Parking State change: {} {}=>{}", id, prevParking, state);
211 + }
212 + // Update instance local state, which includes non-parking state transition
213 + final IntentState prevTransient = transientStates.put(id, state);
214 + log.debug("Transient State change: {} {}=>{}", id, prevTransient, state);
215 +
216 + if (type == null) {
217 + return null;
218 + }
219 + return new IntentEvent(type, intent);
220 + }
221 +
222 + @Override
223 + public void setInstallableIntents(IntentId intentId, List<Intent> result) {
224 + installable.put(intentId, result);
225 + }
226 +
227 + @Override
228 + public List<Intent> getInstallableIntents(IntentId intentId) {
229 + return installable.get(intentId);
230 + }
231 +
232 + @Override
233 + public void removeInstalledIntents(IntentId intentId) {
234 + installable.remove(intentId);
235 + }
236 +
237 + public final class RemoteIntentStateListener extends EntryAdapter<IntentId, IntentState> {
238 +
239 + @Override
240 + public void onEntryEvent(EntryEvent<IntentId, IntentState> event) {
241 + final Member myself = theInstance.getCluster().getLocalMember();
242 + if (!myself.equals(event.getMember())) {
243 + // When Intent state was modified by remote node,
244 + // clear local transient state.
245 + final IntentId intentId = event.getKey();
246 + IntentState oldState = transientStates.remove(intentId);
247 + if (oldState != null) {
248 + log.debug("{} state updated remotely, removing transient state {}",
249 + intentId, oldState);
250 + }
251 + }
252 + }
253 + }
254 +}
...@@ -321,7 +321,7 @@ public class DatabaseStateMachine implements StateMachine { ...@@ -321,7 +321,7 @@ public class DatabaseStateMachine implements StateMachine {
321 private final Map<String, Map<String, VersionedValue>> tableData = Maps.newHashMap(); 321 private final Map<String, Map<String, VersionedValue>> tableData = Maps.newHashMap();
322 private long versionCounter = 1; 322 private long versionCounter = 1;
323 323
324 - public Map<String, VersionedValue> getTable(String tableName) { 324 + Map<String, VersionedValue> getTable(String tableName) {
325 return tableData.get(tableName); 325 return tableData.get(tableName);
326 } 326 }
327 327
......
...@@ -55,6 +55,7 @@ public class MapDBLog implements Log { ...@@ -55,6 +55,7 @@ public class MapDBLog implements Log {
55 .mmapFileEnableIfSupported() 55 .mmapFileEnableIfSupported()
56 .cacheSize(cacheSize) 56 .cacheSize(cacheSize)
57 .makeTxMaker(); 57 .makeTxMaker();
58 + log.info("Raft log file: {}", dbFile.getCanonicalPath());
58 } 59 }
59 60
60 @Override 61 @Override
......