ONOS-2026: Address polling issue in LeaderElection

Change-Id: Ib5c94d932de6b2c3419b07a97d6fe91d5c588538

ONOS-2026: Address polling issue in LeaderElection
Change-Id: Ib5c94d932de6b2c3419b07a97d6fe91d5c588538
Madan Jampani · Gerrit Code Review
Commit 7d243db855023ceafe91a33746b47a36a89e859f 7d243db8 1 parent 3a704316
Showing 1 changed file with 61 additions and 51 deletions
core/store/dist/src/main/java/org/onosproject/store/consistent/impl/DistributedLeadershipManager.java
--- a/core/store/dist/src/main/java/org/onosproject/store/consistent/impl/DistributedLeadershipManager.java
View file @7d243db
+++ b/core/store/dist/src/main/java/org/onosproject/store/consistent/impl/DistributedLeadershipManager.java
View file @7d243db
@@ -84,7 +84,8 @@ public class DistributedLeadershipManager implements LeadershipService {
     private final Logger log = getLogger(getClass());
     private ExecutorService messageHandlingExecutor;
-    private ScheduledExecutorService retryLeaderLockExecutor;
+    private ScheduledExecutorService electionRunner;
+    private ScheduledExecutorService lockExecutor;
     private ScheduledExecutorService staleLeadershipPurgeExecutor;
     private ScheduledExecutorService leadershipStatusBroadcaster;
@@ -98,6 +99,7 @@ public class DistributedLeadershipManager implements LeadershipService {
     private NodeId localNodeId;
     private Set<String> activeTopics = Sets.newConcurrentHashSet();
+    private Map<String, CompletableFuture<Leadership>> pendingFutures = Maps.newConcurrentMap();
     private static final int ELECTION_JOIN_ATTEMPT_INTERVAL_SEC = 2;
     private static final int DELAY_BETWEEN_LEADER_LOCK_ATTEMPTS_SEC = 2;
@@ -125,7 +127,9 @@ public class DistributedLeadershipManager implements LeadershipService {
         messageHandlingExecutor = Executors.newSingleThreadExecutor(
                 groupedThreads("onos/store/leadership", "message-handler"));
-        retryLeaderLockExecutor = Executors.newScheduledThreadPool(
+        electionRunner = Executors.newSingleThreadScheduledExecutor(
+                groupedThreads("onos/store/leadership", "election-runner"));
+        lockExecutor = Executors.newScheduledThreadPool(
                 4, groupedThreads("onos/store/leadership", "election-thread-%d"));
         staleLeadershipPurgeExecutor = Executors.newSingleThreadScheduledExecutor(
                 groupedThreads("onos/store/leadership", "stale-leadership-evictor"));
@@ -139,6 +143,9 @@ public class DistributedLeadershipManager implements LeadershipService {
         clusterService.addListener(clusterEventListener);
+        electionRunner.scheduleWithFixedDelay(
+                this::electLeaders, 0, DELAY_BETWEEN_LEADER_LOCK_ATTEMPTS_SEC, TimeUnit.SECONDS);
+
         leadershipStatusBroadcaster.scheduleWithFixedDelay(
                 this::sendLeadershipStatus, 0, LEADERSHIP_STATUS_UPDATE_INTERVAL_SEC, TimeUnit.SECONDS);
@@ -160,8 +167,9 @@ public class DistributedLeadershipManager implements LeadershipService {
         eventDispatcher.removeSink(LeadershipEvent.class);
         clusterCommunicator.removeSubscriber(LEADERSHIP_EVENT_MESSAGE_SUBJECT);
+        electionRunner.shutdown();
         messageHandlingExecutor.shutdown();
-        retryLeaderLockExecutor.shutdown();
+        lockExecutor.shutdown();
         staleLeadershipPurgeExecutor.shutdown();
         leadershipStatusBroadcaster.shutdown();
@@ -236,7 +244,12 @@ public class DistributedLeadershipManager implements LeadershipService {
                             candidates.creationTime())));
             log.debug("In the leadership race for topic {} with candidates {}", path, candidates);
             activeTopics.add(path);
-            tryLeaderLock(path, future);
+            Leadership leadership = electLeader(path, candidates.value());
+            if (leadership == null) {
+                pendingFutures.put(path, future);
+            } else {
+                future.complete(leadership);
+            }
         } catch (ConsistentMapException e) {
             log.debug("Failed to enter topic leader race for {}. Retrying.", path, e);
             rerunForLeadership(path, future);
@@ -315,7 +328,6 @@ public class DistributedLeadershipManager implements LeadershipService {
                                 localNodeId,
                                 leader.version(),
                                 leader.creationTime())));
-                    retryLock(path, new CompletableFuture<>());
                     return true;
                 }
             }
@@ -355,50 +367,55 @@ public class DistributedLeadershipManager implements LeadershipService {
         return true;
     }
-    private void tryLeaderLock(String path, CompletableFuture<Leadership> future) {
+    private Leadership electLeader(String path, List<NodeId> candidates) {
-        if (!activeTopics.contains(path) || Objects.equals(localNodeId, getLeader(path))) {
+        Leadership currentLeadership = getLeadership(path);
-            return;
+        if (currentLeadership != null) {
-        }
+            return currentLeadership;
-        try {
+        } else {
-            Versioned<List<NodeId>> candidates = candidateMap.get(path);
+            NodeId topCandidate = candidates
-            if (candidates != null) {
+                        .stream()
-                List<NodeId> activeNodes = candidates.value()
+                        .filter(n -> clusterService.getState(n) == ACTIVE)
-                                  .stream()
+                        .findFirst()
-                                  .filter(n -> clusterService.getState(n) == ACTIVE)
+                        .orElse(null);
-                                  .collect(Collectors.toList());
+            try {
-                if (localNodeId.equals(activeNodes.get(LEADER_CANDIDATE_POS))) {
+                Versioned<NodeId> leader = localNodeId.equals(topCandidate)
-                    leaderLockAttempt(path, candidates.value(), future);
+                        ? leaderMap.computeIfAbsent(path, p -> localNodeId) : leaderMap.get(path);
-                } else {
+                if (leader != null) {
-                    retryLock(path, future);
+                    Leadership newLeadership = new Leadership(path,
+                            leader.value(),
+                            leader.version(),
+                            leader.creationTime());
+                    publish(new LeadershipEvent(
+                            LeadershipEvent.Type.LEADER_ELECTED,
+                            newLeadership));
+                    return newLeadership;
                 }
-            } else {
+            } catch (Exception e) {
-                throw new IllegalStateException("should not be here");
+                log.debug("Failed to elect leader for {}", path, e);
             }
-        } catch (Exception e) {
-            log.debug("Failed to fetch candidate information for {}", path, e);
-            retryLock(path, future);
         }
+        return null;
     }
-    private void leaderLockAttempt(String path, List<NodeId> candidates, CompletableFuture<Leadership> future) {
+    private void electLeaders() {
         try {
-            Versioned<NodeId> leader = leaderMap.computeIfAbsent(path, p -> localNodeId);
+            candidateMap.entrySet().forEach(entry -> {
-            if (Objects.equals(leader.value(), localNodeId)) {
+                String path = entry.getKey();
-                log.debug("Assumed leadership for {}", path);
+                List<NodeId> candidates = entry.getValue().value();
-                Leadership leadership = new Leadership(path,
+                if (activeTopics.contains(path)) {
-                        leader.value(),
+                    lockExecutor.submit(() -> {
-                        leader.version(),
+                        Leadership leadership = electLeader(path, candidates);
-                        leader.creationTime());
+                        if (leadership != null) {
-                future.complete(leadership);
+                            CompletableFuture<Leadership> future = pendingFutures.remove(path);
-                publish(new LeadershipEvent(
+                            if (future != null) {
-                        LeadershipEvent.Type.LEADER_ELECTED,
+                                future.complete(leadership);
-                        leadership));
+                            }
-            } else {
+                        }
-                retryLock(path, future);
+                    });
-            }
+                }
+            });
         } catch (Exception e) {
-            log.debug("Attempt to acquire leadership lock for topic {} failed", path, e);
+            log.debug("Failure electing leaders", e);
-            retryLock(path, future);
         }
     }
@@ -451,21 +468,14 @@ public class DistributedLeadershipManager implements LeadershipService {
     }
     private void rerunForLeadership(String path, CompletableFuture<Leadership> future) {
-        retryLeaderLockExecutor.schedule(
+        lockExecutor.schedule(
                 () -> doRunForLeadership(path, future),
                 ELECTION_JOIN_ATTEMPT_INTERVAL_SEC,
                 TimeUnit.SECONDS);
     }
-    private void retryLock(String path, CompletableFuture<Leadership> future) {
-        retryLeaderLockExecutor.schedule(
-                () -> tryLeaderLock(path, future),
-                DELAY_BETWEEN_LEADER_LOCK_ATTEMPTS_SEC,
-                TimeUnit.SECONDS);
-    }
-
     private void retryWithdraw(String path, CompletableFuture<Void> future) {
-        retryLeaderLockExecutor.schedule(
+        lockExecutor.schedule(
                 () -> doWithdraw(path, future),
                 DELAY_BETWEEN_LEADER_LOCK_ATTEMPTS_SEC,
                 TimeUnit.SECONDS);