홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Pavlin Radoslavov
a7243cca 1 parent 255b65fd

Updates for SDN-IP: 

 * Use the new Leadership Service instead of Distributed Lock to
   elect the SDN-IP Leader
 * Reimplement the SDN-IP Intent Synchronizer. In the new implementation
   the Point-to-Point Peer intents are also synchronized by and pushed
   only by the Leader (same as the Multipoint-to-SinglePoint Route intents)
 * Minor cleanups

Change-Id: I8e142781211a1d0f2d362875bc28fd05d843cd4b
Expand all
Inline Side-by-side
Showing 11 changed files with 135 additions and 138 deletions
......@@ -49,6 +49,12 @@
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.0</version>
</dependency>
<dependency>
<groupId>org.onlab.onos</groupId>
<artifactId>onlab-thirdparty</artifactId>
</dependency>
......@@ -69,10 +75,12 @@
<artifactId>onos-cli</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.karaf.shell</groupId>
<artifactId>org.apache.karaf.shell.console</artifactId>
</dependency>
<dependency>
<groupId>org.osgi</groupId>
<artifactId>org.osgi.core</artifactId>
......
......@@ -16,6 +16,7 @@
package org.onlab.onos.sdnip;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.onlab.onos.core.ApplicationId;
......@@ -24,8 +25,6 @@ import org.onlab.onos.net.flow.DefaultTrafficSelector;
import org.onlab.onos.net.flow.DefaultTrafficTreatment;
import org.onlab.onos.net.flow.TrafficSelector;
import org.onlab.onos.net.flow.TrafficTreatment;
import org.onlab.onos.net.intent.Intent;
import org.onlab.onos.net.intent.IntentService;
import org.onlab.onos.net.intent.PointToPointIntent;
import org.onlab.onos.sdnip.bgp.BgpConstants;
import org.onlab.onos.sdnip.config.BgpPeer;
......@@ -48,9 +47,9 @@ public class PeerConnectivityManager {
private static final Logger log = LoggerFactory.getLogger(
PeerConnectivityManager.class);
private final IntentSynchronizer intentSynchronizer;
private final SdnIpConfigService configService;
private final InterfaceService interfaceService;
private final IntentService intentService;
private final ApplicationId appId;
......@@ -58,18 +57,18 @@ public class PeerConnectivityManager {
* Creates a new PeerConnectivityManager.
*
* @param appId the application ID
* @param intentSynchronizer the intent synchronizer
* @param configService the SDN-IP config service
* @param interfaceService the interface service
* @param intentService the intent service
*/
public PeerConnectivityManager(ApplicationId appId,
IntentSynchronizer intentSynchronizer,
SdnIpConfigService configService,
InterfaceService interfaceService,
IntentService intentService) {
InterfaceService interfaceService) {
this.appId = appId;
this.intentSynchronizer = intentSynchronizer;
this.configService = configService;
this.interfaceService = interfaceService;
this.intentService = intentService;
}
/**
......@@ -107,6 +106,8 @@ public class PeerConnectivityManager {
* {@link BgpSpeaker}s and all external {@link BgpPeer}s.
*/
private void setUpConnectivity() {
List<PointToPointIntent> intents = new ArrayList<>();
for (BgpSpeaker bgpSpeaker : configService.getBgpSpeakers()
.values()) {
log.debug("Start to set up BGP paths for BGP speaker: {}",
......@@ -117,9 +118,12 @@ public class PeerConnectivityManager {
log.debug("Start to set up BGP paths between BGP speaker: {} "
+ "to BGP peer: {}", bgpSpeaker, bgpPeer);
buildPeerIntents(bgpSpeaker, bgpPeer);
intents.addAll(buildPeerIntents(bgpSpeaker, bgpPeer));
}
}
// Submit all the intents.
intentSynchronizer.submitPeerIntents(intents);
}
/**
......@@ -128,9 +132,12 @@ public class PeerConnectivityManager {
*
* @param bgpSpeaker the BGP speaker
* @param bgpPeer the BGP peer
* @return the intents to install
*/
private void buildPeerIntents(BgpSpeaker bgpSpeaker, BgpPeer bgpPeer) {
List<Intent> intents = new ArrayList<Intent>();
private Collection<PointToPointIntent> buildPeerIntents(
BgpSpeaker bgpSpeaker,
BgpPeer bgpPeer) {
List<PointToPointIntent> intents = new ArrayList<>();
ConnectPoint bgpdConnectPoint = bgpSpeaker.connectPoint();
......@@ -142,7 +149,7 @@ public class PeerConnectivityManager {
if (peerInterface == null) {
log.error("No interface found for peer {}", bgpPeer.ipAddress());
return;
return intents;
}
IpAddress bgpdAddress = null;
......@@ -156,7 +163,7 @@ public class PeerConnectivityManager {
if (bgpdAddress == null) {
log.debug("No IP address found for peer {} on interface {}",
bgpPeer, bgpPeer.connectPoint());
return;
return intents;
}
IpAddress bgpdPeerAddress = bgpPeer.ipAddress();
......@@ -231,11 +238,7 @@ public class PeerConnectivityManager {
intents.add(new PointToPointIntent(appId, selector, treatment,
bgpdPeerConnectPoint, bgpdConnectPoint));
// Submit all the intents.
// TODO submit as a batch
for (Intent intent : intents) {
intentService.submit(intent);
}
return intents;
}
/**
......@@ -249,7 +252,8 @@ public class PeerConnectivityManager {
* @return the new traffic selector
*/
private TrafficSelector buildSelector(byte ipProto, IpAddress srcIp,
IpAddress dstIp, Short srcTcpPort, Short dstTcpPort) {
IpAddress dstIp, Short srcTcpPort,
Short dstTcpPort) {
TrafficSelector.Builder builder = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
.matchIPProtocol(ipProto)
......
......@@ -92,18 +92,19 @@ public class Router implements RouteListener {
*
* @param appId the application ID
* @param intentSynchronizer the intent synchronizer
* @param hostService the host service
* @param configService the configuration service
* @param interfaceService the interface service
* @param hostService the host service
*/
public Router(ApplicationId appId, IntentSynchronizer intentSynchronizer,
HostService hostService, SdnIpConfigService configService,
InterfaceService interfaceService) {
SdnIpConfigService configService,
InterfaceService interfaceService,
HostService hostService) {
this.appId = appId;
this.intentSynchronizer = intentSynchronizer;
this.hostService = hostService;
this.configService = configService;
this.interfaceService = interfaceService;
this.hostService = hostService;
this.hostListener = new InternalHostListener();
......
......@@ -18,8 +18,6 @@ package org.onlab.onos.sdnip;
import static org.slf4j.LoggerFactory.getLogger;
import java.util.Collection;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
......@@ -27,6 +25,11 @@ import org.apache.felix.scr.annotations.Deactivate;
import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.apache.felix.scr.annotations.Service;
import org.onlab.onos.cluster.ClusterService;
import org.onlab.onos.cluster.ControllerNode;
import org.onlab.onos.cluster.LeadershipEvent;
import org.onlab.onos.cluster.LeadershipEventListener;
import org.onlab.onos.cluster.LeadershipService;
import org.onlab.onos.core.ApplicationId;
import org.onlab.onos.core.CoreService;
import org.onlab.onos.net.host.HostService;
......@@ -35,11 +38,8 @@ import org.onlab.onos.sdnip.bgp.BgpRouteEntry;
import org.onlab.onos.sdnip.bgp.BgpSession;
import org.onlab.onos.sdnip.bgp.BgpSessionManager;
import org.onlab.onos.sdnip.config.SdnIpConfigReader;
import org.onlab.onos.store.service.Lock;
import org.onlab.onos.store.service.LockService;
import org.slf4j.Logger;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.slf4j.Logger;
/**
* Component for the SDN-IP peering application.
......@@ -65,55 +65,49 @@ public class SdnIp implements SdnIpService {
protected HostService hostService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected LockService lockService;
protected ClusterService clusterService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected LeadershipService leadershipService;
private IntentSynchronizer intentSynchronizer;
private SdnIpConfigReader config;
private PeerConnectivityManager peerConnectivity;
private Router router;
private BgpSessionManager bgpSessionManager;
private ExecutorService leaderElectionExecutor;
private Lock leaderLock;
private volatile boolean isShutdown = true;
private LeadershipEventListener leadershipEventListener =
new InnerLeadershipEventListener();
ApplicationId appId;
private ControllerNode localControllerNode;
@Activate
protected void activate() {
log.info("SDN-IP started");
isShutdown = false;
ApplicationId appId = coreService.registerApplication(SDN_IP_APP);
appId = coreService.registerApplication(SDN_IP_APP);
config = new SdnIpConfigReader();
config.init();
localControllerNode = clusterService.getLocalNode();
InterfaceService interfaceService =
new HostToInterfaceAdaptor(hostService);
intentSynchronizer = new IntentSynchronizer(appId, intentService);
intentSynchronizer.start();
peerConnectivity = new PeerConnectivityManager(appId, config,
interfaceService, intentService);
peerConnectivity = new PeerConnectivityManager(appId,
intentSynchronizer,
config,
interfaceService);
peerConnectivity.start();
router = new Router(appId, intentSynchronizer, hostService, config,
interfaceService);
router = new Router(appId, intentSynchronizer, config,
interfaceService, hostService);
router.start();
leaderLock = lockService.create(SDN_IP_APP + "/sdnIpLeaderLock");
leaderElectionExecutor = Executors.newSingleThreadExecutor(
new ThreadFactoryBuilder()
.setNameFormat("sdnip-leader-election-%d").build());
leaderElectionExecutor.execute(new Runnable() {
@Override
public void run() {
doLeaderElectionThread();
}
});
// Manually set the instance as the leader to allow testing
// TODO change this when we get a leader election
// intentSynchronizer.leaderChanged(true);
leadershipService.addListener(leadershipEventListener);
leadershipService.runForLeadership(appId.name());
bgpSessionManager = new BgpSessionManager(router);
// TODO: the local BGP listen port number should be configurable
......@@ -124,17 +118,16 @@ public class SdnIp implements SdnIpService {
@Deactivate
protected void deactivate() {
isShutdown = true;
bgpSessionManager.stop();
router.stop();
peerConnectivity.stop();
intentSynchronizer.stop();
// Stop the thread(s)
leaderElectionExecutor.shutdownNow();
leadershipService.withdraw(appId.name());
leadershipService.removeListener(leadershipEventListener);
log.info("Stopped");
log.info("SDN-IP Stopped");
}
@Override
......@@ -162,63 +155,38 @@ public class SdnIp implements SdnIpService {
}
/**
* Performs the leader election.
* A listener for Leadership Events.
*/
private void doLeaderElectionThread() {
private class InnerLeadershipEventListener
implements LeadershipEventListener {
//
// Try to acquire the lock and keep extending it until the instance
// is shutdown.
//
while (!isShutdown) {
log.debug("SDN-IP Leader Election begin");
@Override
public void event(LeadershipEvent event) {
log.debug("Leadership Event: time = {} type = {} event = {}",
event.time(), event.type(), event);
// Block until it becomes the leader
try {
leaderLock.lock(LEASE_DURATION_MS);
if (!event.subject().topic().equals(appId.name())) {
return; // Not our topic: ignore
}
if (!event.subject().leader().id().equals(
localControllerNode.id())) {
return; // The event is not about this instance: ignore
}
// This instance is the leader
switch (event.type()) {
case LEADER_ELECTED:
log.info("SDN-IP Leader Elected");
intentSynchronizer.leaderChanged(true);
// Keep extending the expiration until shutdown
int extensionFailedCountdown = LEASE_EXTEND_RETRY_MAX - 1;
//
// Keep periodically extending the lock expiration.
// If there are multiple back-to-back failures to extend (with
// extra sleep time between retrials), then release the lock.
//
while (!isShutdown) {
Thread.sleep(LEASE_DURATION_MS / LEASE_EXTEND_RETRY_MAX);
if (leaderLock.extendExpiration(LEASE_DURATION_MS)) {
log.trace("SDN-IP Leader Extended");
extensionFailedCountdown = LEASE_EXTEND_RETRY_MAX;
} else {
log.debug("SDN-IP Leader Cannot Extend Election");
if (!leaderLock.isLocked()) {
log.debug("SDN-IP Leader Lock Lost");
intentSynchronizer.leaderChanged(false);
break; // Try again to get the lock
}
extensionFailedCountdown--;
if (extensionFailedCountdown <= 0) {
// Failed too many times to extend.
// Release the lock.
log.debug("SDN-IP Leader Lock Released");
intentSynchronizer.leaderChanged(false);
leaderLock.unlock();
break; // Try again to get the lock
}
}
}
} catch (InterruptedException e) {
// Thread interrupted. Time to shutdown
log.debug("SDN-IP Leader Interrupted");
break;
case LEADER_BOOTED:
log.info("SDN-IP Leader Lost Election");
intentSynchronizer.leaderChanged(false);
break;
case LEADER_REELECTED:
break;
default:
break;
}
}
// If we reach here, the instance was shutdown
intentSynchronizer.leaderChanged(false);
leaderLock.unlock();
}
}
......
......@@ -20,6 +20,8 @@ import org.easymock.IArgumentMatcher;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.onlab.junit.TestUtils;
import org.onlab.junit.TestUtils.TestUtilsException;
import org.onlab.onos.core.ApplicationId;
import org.onlab.onos.net.ConnectPoint;
import org.onlab.onos.net.DeviceId;
......@@ -70,9 +72,10 @@ public class PeerConnectivityManagerTest {
};
private PeerConnectivityManager peerConnectivityManager;
private IntentService intentService;
private IntentSynchronizer intentSynchronizer;
private SdnIpConfigService configInfoService;
private InterfaceService interfaceService;
private IntentService intentService;
private Map<String, BgpSpeaker> bgpSpeakers;
private Map<String, Interface> interfaces;
......@@ -525,8 +528,10 @@ public class PeerConnectivityManagerTest {
/**
* Initializes peer connectivity testing environment.
*
* @throws TestUtilsException if exceptions when using TestUtils
*/
private void initPeerConnectivity() {
private void initPeerConnectivity() throws TestUtilsException {
configInfoService = createMock(SdnIpConfigService.class);
expect(configInfoService.getBgpPeers()).andReturn(peers).anyTimes();
......@@ -536,8 +541,13 @@ public class PeerConnectivityManagerTest {
intentService = createMock(IntentService.class);
replay(intentService);
peerConnectivityManager = new PeerConnectivityManager(APPID, configInfoService,
interfaceService, intentService);
intentSynchronizer = new IntentSynchronizer(APPID, intentService);
intentSynchronizer.leaderChanged(true);
TestUtils.setField(intentSynchronizer, "isActivatedLeader", true);
peerConnectivityManager =
new PeerConnectivityManager(APPID, intentSynchronizer,
configInfoService, interfaceService);
}
/*
......
......@@ -115,8 +115,8 @@ public class RouterTest {
intentService = createMock(IntentService.class);
intentSynchronizer = new IntentSynchronizer(APPID, intentService);
router = new Router(APPID, intentSynchronizer,
hostService, sdnIpConfigService, interfaceService);
router = new Router(APPID, intentSynchronizer, sdnIpConfigService,
interfaceService, hostService);
}
/**
......@@ -267,8 +267,8 @@ public class RouterTest {
// Verify
assertEquals(router.getRoutes().size(), 1);
assertTrue(router.getRoutes().contains(routeEntry));
assertEquals(intentSynchronizer.getPushedRouteIntents().size(), 1);
assertEquals(intentSynchronizer.getPushedRouteIntents().iterator().next(),
assertEquals(intentSynchronizer.getRouteIntents().size(), 1);
assertEquals(intentSynchronizer.getRouteIntents().iterator().next(),
intent);
verify(intentService);
}
......@@ -347,8 +347,8 @@ public class RouterTest {
// Verify
assertEquals(router.getRoutes().size(), 1);
assertTrue(router.getRoutes().contains(routeEntryUpdate));
assertEquals(intentSynchronizer.getPushedRouteIntents().size(), 1);
assertEquals(intentSynchronizer.getPushedRouteIntents().iterator().next(),
assertEquals(intentSynchronizer.getRouteIntents().size(), 1);
assertEquals(intentSynchronizer.getRouteIntents().iterator().next(),
intentNew);
verify(intentService);
}
......@@ -397,7 +397,7 @@ public class RouterTest {
// Verify
assertEquals(router.getRoutes().size(), 0);
assertEquals(intentSynchronizer.getPushedRouteIntents().size(), 0);
assertEquals(intentSynchronizer.getRouteIntents().size(), 0);
verify(intentService);
}
......@@ -425,7 +425,7 @@ public class RouterTest {
// Verify
assertEquals(router.getRoutes().size(), 1);
assertTrue(router.getRoutes().contains(routeEntry));
assertEquals(intentSynchronizer.getPushedRouteIntents().size(), 0);
assertEquals(intentSynchronizer.getRouteIntents().size(), 0);
verify(intentService);
}
}
......
......@@ -117,7 +117,7 @@ public class RouterTestWithAsyncArp {
intentSynchronizer = new IntentSynchronizer(APPID, intentService);
router = new Router(APPID, intentSynchronizer,
hostService, sdnIpConfigService, interfaceService);
sdnIpConfigService, interfaceService, hostService);
internalHostListener = router.new InternalHostListener();
}
......@@ -229,8 +229,8 @@ public class RouterTestWithAsyncArp {
// Verify
assertEquals(router.getRoutes().size(), 1);
assertTrue(router.getRoutes().contains(routeEntry));
assertEquals(intentSynchronizer.getPushedRouteIntents().size(), 1);
assertEquals(intentSynchronizer.getPushedRouteIntents().iterator().next(),
assertEquals(intentSynchronizer.getRouteIntents().size(), 1);
assertEquals(intentSynchronizer.getRouteIntents().iterator().next(),
intent);
verify(intentService);
verify(hostService);
......@@ -254,9 +254,9 @@ public class RouterTestWithAsyncArp {
MultiPointToSinglePointIntent intent = staticIntentBuilder();
// Set up the bgpRoutes field of Router class with existing route, and
// pushedRouteIntents field with the corresponding existing intent
// routeIntents field with the corresponding existing intent
setBgpRoutesField(routeEntry);
setPushedRouteIntentsField(routeEntry, intent);
setRouteIntentsField(routeEntry, intent);
// Start to construct a new route entry and new intent
RouteEntry routeEntryUpdate = new RouteEntry(
......@@ -312,8 +312,8 @@ public class RouterTestWithAsyncArp {
// Verify
assertEquals(router.getRoutes().size(), 1);
assertTrue(router.getRoutes().contains(routeEntryUpdate));
assertEquals(intentSynchronizer.getPushedRouteIntents().size(), 1);
assertEquals(intentSynchronizer.getPushedRouteIntents().iterator().next(),
assertEquals(intentSynchronizer.getRouteIntents().size(), 1);
assertEquals(intentSynchronizer.getRouteIntents().iterator().next(),
intentNew);
verify(intentService);
verify(hostService);
......@@ -334,9 +334,9 @@ public class RouterTestWithAsyncArp {
MultiPointToSinglePointIntent intent = staticIntentBuilder();
// Set up the bgpRoutes field of Router class with existing route, and
// pushedRouteIntents field with the corresponding existing intent
// routeIntents field with the corresponding existing intent
setBgpRoutesField(routeEntry);
setPushedRouteIntentsField(routeEntry, intent);
setRouteIntentsField(routeEntry, intent);
// Set up expectation
reset(intentService);
......@@ -350,7 +350,7 @@ public class RouterTestWithAsyncArp {
// Verify
assertEquals(router.getRoutes().size(), 0);
assertEquals(intentSynchronizer.getPushedRouteIntents().size(), 0);
assertEquals(intentSynchronizer.getRouteIntents().size(), 0);
verify(intentService);
}
......@@ -397,17 +397,17 @@ public class RouterTestWithAsyncArp {
}
/**
* Sets pushedRouteIntentsField in Router class.
* Sets routeIntentsField in IntentSynchronizer class.
*
* @throws TestUtilsException
*/
private void setPushedRouteIntentsField(RouteEntry routeEntry,
private void setRouteIntentsField(RouteEntry routeEntry,
MultiPointToSinglePointIntent intent)
throws TestUtilsException {
ConcurrentHashMap<Ip4Prefix, MultiPointToSinglePointIntent>
pushedRouteIntents = new ConcurrentHashMap<>();
pushedRouteIntents.put(routeEntry.prefix(), intent);
TestUtils.setField(router, "pushedRouteIntents", pushedRouteIntents);
routeIntents = new ConcurrentHashMap<>();
routeIntents.put(routeEntry.prefix(), intent);
TestUtils.setField(intentSynchronizer, "routeIntents", routeIntents);
}
}
\ No newline at end of file
......
......@@ -113,8 +113,8 @@ public class SdnIpTest {
random = new Random();
intentSynchronizer = new IntentSynchronizer(APPID, intentService);
router = new Router(APPID, intentSynchronizer, hostService,
sdnIpConfigService, interfaceService);
router = new Router(APPID, intentSynchronizer, sdnIpConfigService,
interfaceService, hostService);
}
/**
......@@ -241,7 +241,7 @@ public class SdnIpTest {
latch.await(5000, TimeUnit.MILLISECONDS);
assertEquals(router.getRoutes().size(), numRoutes);
assertEquals(intentSynchronizer.getPushedRouteIntents().size(),
assertEquals(intentSynchronizer.getRouteIntents().size(),
numRoutes);
verify(intentService);
......@@ -317,7 +317,7 @@ public class SdnIpTest {
deleteCount.await(5000, TimeUnit.MILLISECONDS);
assertEquals(0, router.getRoutes().size());
assertEquals(0, intentSynchronizer.getPushedRouteIntents().size());
assertEquals(0, intentSynchronizer.getRouteIntents().size());
verify(intentService);
}
......
......@@ -139,6 +139,12 @@
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.0</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<version>1.9.13</version>
......