Workaround for ONOS-1198

- check before unlocking to avoid Exception
- mark field referenced from multiple threads

Change-Id: I4da8450811aa69b9cc02a5e24178e7293d056aea

@@ -18,6 +18,7 @@ package org.onosproject.store.cluster.impl;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Maps;
 import com.hazelcast.config.TopicConfig;
 import com.hazelcast.config.TopicConfig;
 import com.hazelcast.core.IAtomicLong;
 import com.hazelcast.core.IAtomicLong;
+import com.hazelcast.core.ILock;
 
 
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
@@ -50,7 +51,6 @@ import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.Future;
-import java.util.concurrent.locks.Lock;
 import java.util.stream.Collectors;
 import java.util.stream.Collectors;
 
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkArgument;
@@ -247,8 +247,8 @@ public class HazelcastLeadershipService implements LeadershipService {
         // higher if the mastership has changed any times.
         // higher if the mastership has changed any times.
         private long myLastLeaderTerm = NO_TERM;
         private long myLastLeaderTerm = NO_TERM;
 
 
-        private NodeId leader;
-        private Lock leaderLock;
+        private volatile NodeId leader;
+        private ILock leaderLock;
         private Future<?> getLockFuture;
         private Future<?> getLockFuture;
         private Future<?> periodicProcessingFuture;
         private Future<?> periodicProcessingFuture;
 
 
@@ -427,6 +427,8 @@ public class HazelcastLeadershipService implements LeadershipService {
                             long delta = System.currentTimeMillis() -
                             long delta = System.currentTimeMillis() -
                                 lastLeadershipUpdateMs;
                                 lastLeadershipUpdateMs;
                             if (delta > LEADERSHIP_REMOTE_TIMEOUT_MS) {
                             if (delta > LEADERSHIP_REMOTE_TIMEOUT_MS) {
+                                log.debug("Topic {} leader {} booted due to heartbeat timeout",
+                                          topicName, leader);
                                 leadershipEvent = new LeadershipEvent(
                                 leadershipEvent = new LeadershipEvent(
                                         LeadershipEvent.Type.LEADER_BOOTED,
                                         LeadershipEvent.Type.LEADER_BOOTED,
                                         new Leadership(topicName, leader, myLastLeaderTerm));
                                         new Leadership(topicName, leader, myLastLeaderTerm));
@@ -519,7 +521,9 @@ public class HazelcastLeadershipService implements LeadershipService {
                                         clusterService.getLocalNode().id(),
                                         clusterService.getLocalNode().id(),
                                         LEADERSHIP_EVENT_MESSAGE_SUBJECT,
                                         LEADERSHIP_EVENT_MESSAGE_SUBJECT,
                                         SERIALIZER.encode(leadershipEvent)));
                                         SERIALIZER.encode(leadershipEvent)));
-                        leaderLock.unlock();
+                        if (leaderLock.isLockedByCurrentThread()) {
+                            leaderLock.unlock();
+                        }
                     }
                     }
                 }
                 }
             }
             }