[ONOS-4482] Implement dynamic add or remove a gateway node

 - Implements command line interface for dynamic management
 - Implements dynamic gateway node management
 - Add CLIs

Change-Id: I27bb945968a262d2813d317fc79e75ee768b2825

 COMPILE_DEPS = [
 COMPILE_DEPS = [
     '//lib:CORE_DEPS',
     '//lib:CORE_DEPS',
+    '//lib:org.apache.karaf.shell.console',
+    '//cli:onos-cli',
+    '//core/store/serializers:onos-core-serializers',
 ]
 ]
 
 
 osgi_jar_with_tests (
 osgi_jar_with_tests (

@@ -53,5 +53,15 @@
             <artifactId>onos-core-serializers</artifactId>
             <artifactId>onos-core-serializers</artifactId>
             <version>${project.version}</version>
             <version>${project.version}</version>
         </dependency>
         </dependency>
+        <dependency>
+            <groupId>org.apache.karaf.shell</groupId>
+            <artifactId>org.apache.karaf.shell.console</artifactId>
+            <version>3.0.5</version>
+        </dependency>
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-cli</artifactId>
+            <version>1.7.0-SNAPSHOT</version>
+        </dependency>
     </dependencies>
     </dependencies>
 </project>
 </project>

+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.scalablegateway.cli;
+
+import com.google.common.collect.Lists;
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.packet.Ip4Address;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.DeviceId;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Adds gateway node information for scalablegateway node managements.
+ */
+
+@Command(scope = "onos", name = "gateway-add",
+        description = "Adds gateway node information for scalablegateway node managements")
+public class ScalableGatewayAddCommand extends AbstractShellCommand {
+
+    private static final String SUCCESS = "Process of adding gateway node is succeed";
+    private static final String FAIL = "Process of adding gateway node is failed";
+
+    @Argument(index = 0, name = "DeviceId", description = "GatewayNode device id",
+            required = true, multiValued = false)
+    String deviceId = null;
+
+    @Argument(index = 1, name = "dataPlaneIp",
+            description = "GatewayNode datePlane interface ip address",
+            required = true, multiValued = false)
+    String ipAddress = null;
+
+    @Argument(index = 2, name = "extInterfaceNames",
+            description = "GatewayNode Interface name to outgoing external network",
+            required = true, multiValued = true)
+    String interfaceName = null;
+
+    @Override
+    protected void execute() {
+        ScalableGatewayService service = get(ScalableGatewayService.class);
+
+        GatewayNode gatewayNode = GatewayNode.builder()
+                .gatewayDeviceId(DeviceId.deviceId(deviceId))
+                .dataIpAddress(Ip4Address.valueOf(ipAddress))
+                .gatewayExternalInterfaceNames(splitNameList(interfaceName))
+                .build();
+        if (service.addGatewayNode(gatewayNode)) {
+            print(SUCCESS);
+        } else {
+            print(FAIL);
+        }
+    }
+
+    private List<String> splitNameList(String interfaceName) {
+        List<String> list = Lists.newArrayList();
+        return Collections.addAll(list, interfaceName.split(",")) ? list : null;
+    }
+}

+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.scalablegateway.cli;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.DeviceId;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
+
+/**
+ * Deletes gateway node information for scalablegateway node managements.
+ */
+
+@Command(scope = "onos", name = "gateway-delete",
+        description = "Deletes gateway node information for scalablegateway node managements")
+public class ScalableGatewayDeleteCommand extends AbstractShellCommand {
+
+    private static final String SUCCESS = "Process of deleting gateway node is succeed.";
+    private static final String FAIL = "Process of deleting gateway node is failed.";
+    private static final String UNKNOWN = "Unknown device id is given.";
+
+    @Argument(index = 0, name = "DeviceId", description = "GatewayNode device id",
+            required = true, multiValued = false)
+    String deviceId = null;
+
+    @Override
+    protected void execute() {
+        ScalableGatewayService service = get(ScalableGatewayService.class);
+
+        GatewayNode gatewayNode = service.getGatewayNode(DeviceId.deviceId(deviceId));
+        if (gatewayNode == null) {
+            print(UNKNOWN);
+            return;
+        }
+
+        if (service.deleteGatewayNode(gatewayNode)) {
+            print(SUCCESS);
+        } else {
+            print(FAIL);
+        }
+    }
+
+}

+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.scalablegateway.cli;
+
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
+
+/**
+ * Lists all gateway node information of scalablegateway.
+ */
+
+@Command(scope = "onos", name = "gateways",
+        description = "Lists gateway node information")
+public class ScalableGatewayListCommand extends AbstractShellCommand {
+
+    private static final String FORMAT = "GatewayNode Id[%s]: DataPlane Ip[%s], External Interface names[%s]";
+    @Override
+    protected void execute() {
+        ScalableGatewayService service = get(ScalableGatewayService.class);
+        service.getGatewayNodes().forEach(node -> print(FORMAT,
+                node.getGatewayDeviceId().toString(),
+                node.getDataIpAddress().toString(),
+                node.getGatewayExternalInterfaceNames().toString()));
+    }
+}

+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Command line interface for Scaleable Gateway management.
+ */
+package org.onosproject.scalablegateway.cli;
\ No newline at end of file

@@ -17,7 +17,6 @@
 package org.onosproject.scalablegateway.impl;
 package org.onosproject.scalablegateway.impl;
 
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -25,6 +24,7 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.Service;
 import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.KryoNamespace;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.CoreService;
 import org.onosproject.core.CoreService;
 
 
@@ -38,6 +38,8 @@ import org.onosproject.net.config.NetworkConfigListener;
 import org.onosproject.net.config.NetworkConfigRegistry;
 import org.onosproject.net.config.NetworkConfigRegistry;
 import org.onosproject.net.config.NetworkConfigService;
 import org.onosproject.net.config.NetworkConfigService;
 import org.onosproject.net.config.basics.SubjectFactories;
 import org.onosproject.net.config.basics.SubjectFactories;
+import org.onosproject.net.device.DeviceEvent;
+import org.onosproject.net.device.DeviceListener;
 import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.driver.DriverService;
 import org.onosproject.net.driver.DriverService;
 import org.onosproject.net.group.Group;
 import org.onosproject.net.group.Group;
@@ -48,8 +50,12 @@ import org.onosproject.scalablegateway.api.GatewayNodeConfig;
 import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.onosproject.scalablegateway.api.ScalableGatewayService;
 
 
 import java.util.List;
 import java.util.List;
-import java.util.Map;
 
 
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
+import org.onosproject.store.service.Versioned;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;
 
 
@@ -72,6 +78,7 @@ public class ScalableGatewayManager implements ScalableGatewayService {
     private static final String FAIL_ADD_GATEWAY = "Adding process is failed as existing deivce id";
     private static final String FAIL_ADD_GATEWAY = "Adding process is failed as existing deivce id";
     private static final String FAIL_REMOVE_GATEWAY = "Removing process is failed as unknown deivce id";
     private static final String FAIL_REMOVE_GATEWAY = "Removing process is failed as unknown deivce id";
     private static final String PORT_NAME = "portName";
     private static final String PORT_NAME = "portName";
+    private static final String GATEWAYNODE_MAP_NAME = "gatewaynode-map";
 
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected CoreService coreService;
     protected CoreService coreService;
@@ -91,10 +98,14 @@ public class ScalableGatewayManager implements ScalableGatewayService {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected GroupService groupService;
     protected GroupService groupService;
 
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected StorageService storageService;
+
     private GatewayNodeConfig config;
     private GatewayNodeConfig config;
     private SelectGroupHandler selectGroupHandler;
     private SelectGroupHandler selectGroupHandler;
 
 
     private final NetworkConfigListener configListener = new InternalConfigListener();
     private final NetworkConfigListener configListener = new InternalConfigListener();
+    private InternalDeviceListener internalDeviceListener = new InternalDeviceListener();
 
 
     private final ConfigFactory configFactory =
     private final ConfigFactory configFactory =
             new ConfigFactory(SubjectFactories.APP_SUBJECT_FACTORY, GatewayNodeConfig.class, APP_NAME) {
             new ConfigFactory(SubjectFactories.APP_SUBJECT_FACTORY, GatewayNodeConfig.class, APP_NAME) {
@@ -103,17 +114,28 @@ public class ScalableGatewayManager implements ScalableGatewayService {
                     return new GatewayNodeConfig();
                     return new GatewayNodeConfig();
                 }
                 }
             };
             };
-    private Map<DeviceId, GatewayNode> gatewayNodeMap = Maps.newHashMap(); // Map<GatewayNode`s Id, GatewayNode object>
+    private ConsistentMap<DeviceId, GatewayNode> gatewayNodeMap; // Map<GatewayNode`s Id, GatewayNode object>
+    private static final KryoNamespace.Builder GATEWAYNODE_SERIALIZER = KryoNamespace.newBuilder()
+            .register(KryoNamespaces.API)
+            .register(DeviceId.class)
+            .register(GatewayNode.class);
 
 
     @Activate
     @Activate
     protected void activate() {
     protected void activate() {
         appId = coreService.registerApplication(APP_ID);
         appId = coreService.registerApplication(APP_ID);
         configRegistry.registerConfigFactory(configFactory);
         configRegistry.registerConfigFactory(configFactory);
         configService.addListener(configListener);
         configService.addListener(configListener);
+        deviceService.addListener(internalDeviceListener);
 
 
         selectGroupHandler = new SelectGroupHandler(groupService, deviceService, driverService, appId);
         selectGroupHandler = new SelectGroupHandler(groupService, deviceService, driverService, appId);
         readConfiguration();
         readConfiguration();
 
 
+        gatewayNodeMap = storageService.<DeviceId, GatewayNode>consistentMapBuilder()
+                .withSerializer(Serializer.using(GATEWAYNODE_SERIALIZER.build()))
+                .withName(GATEWAYNODE_MAP_NAME)
+                .withApplicationId(appId)
+                .build();
+
         log.info("started");
         log.info("started");
     }
     }
 
 
@@ -121,6 +143,7 @@ public class ScalableGatewayManager implements ScalableGatewayService {
     protected void deactivate() {
     protected void deactivate() {
         gatewayNodeMap.clear();
         gatewayNodeMap.clear();
 
 
+        deviceService.removeListener(internalDeviceListener);
         configService.removeListener(configListener);
         configService.removeListener(configListener);
 
 
         log.info("stopped");
         log.info("stopped");
@@ -128,12 +151,12 @@ public class ScalableGatewayManager implements ScalableGatewayService {
 
 
     @Override
     @Override
     public GatewayNode getGatewayNode(DeviceId deviceId) {
     public GatewayNode getGatewayNode(DeviceId deviceId) {
-        return checkNotNull(gatewayNodeMap.get(deviceId), GATEWAYNODE_CAN_NOT_BE_NULL);
+        return checkNotNull(gatewayNodeMap.get(deviceId).value(), GATEWAYNODE_CAN_NOT_BE_NULL);
     }
     }
 
 
     @Override
     @Override
     public List<PortNumber> getGatewayExternalPorts(DeviceId deviceId) {
     public List<PortNumber> getGatewayExternalPorts(DeviceId deviceId) {
-        GatewayNode gatewayNode = checkNotNull(gatewayNodeMap.get(deviceId), GATEWAYNODE_CAN_NOT_BE_NULL);
+        GatewayNode gatewayNode = checkNotNull(gatewayNodeMap.get(deviceId).value(), GATEWAYNODE_CAN_NOT_BE_NULL);
         List<PortNumber> portNumbers = Lists.newArrayList();
         List<PortNumber> portNumbers = Lists.newArrayList();
         gatewayNode.getGatewayExternalInterfaceNames()
         gatewayNode.getGatewayExternalInterfaceNames()
                 .stream()
                 .stream()
@@ -164,6 +187,7 @@ public class ScalableGatewayManager implements ScalableGatewayService {
         List<GatewayNode> gatewayNodeList = Lists.newArrayList();
         List<GatewayNode> gatewayNodeList = Lists.newArrayList();
         gatewayNodeMap.values()
         gatewayNodeMap.values()
                 .stream()
                 .stream()
+                .map(Versioned::value)
                 .forEach(gatewayNode -> gatewayNodeList.add(gatewayNode));
                 .forEach(gatewayNode -> gatewayNodeList.add(gatewayNode));
         return gatewayNodeList;
         return gatewayNodeList;
 
 
@@ -174,6 +198,7 @@ public class ScalableGatewayManager implements ScalableGatewayService {
         List<DeviceId> deviceIdList = Lists.newArrayList();
         List<DeviceId> deviceIdList = Lists.newArrayList();
         gatewayNodeMap.values()
         gatewayNodeMap.values()
                 .stream()
                 .stream()
+                .map(Versioned::value)
                 .forEach(gatewayNode -> deviceIdList.add(gatewayNode.getGatewayDeviceId()));
                 .forEach(gatewayNode -> deviceIdList.add(gatewayNode.getGatewayDeviceId()));
         return deviceIdList;
         return deviceIdList;
 
 
@@ -182,12 +207,24 @@ public class ScalableGatewayManager implements ScalableGatewayService {
     @Override
     @Override
     public boolean addGatewayNode(GatewayNode gatewayNode) {
     public boolean addGatewayNode(GatewayNode gatewayNode) {
         gatewayNodeMap.putIfAbsent(gatewayNode.getGatewayDeviceId(), gatewayNode);
         gatewayNodeMap.putIfAbsent(gatewayNode.getGatewayDeviceId(), gatewayNode);
+        updateGatewayLoadBalance(gatewayNode, true);
         return true;
         return true;
     }
     }
 
 
     @Override
     @Override
     public boolean deleteGatewayNode(GatewayNode gatewayNode) {
     public boolean deleteGatewayNode(GatewayNode gatewayNode) {
-        return gatewayNodeMap.remove(gatewayNode.getGatewayDeviceId(), gatewayNode);
+        boolean result = gatewayNodeMap.remove(gatewayNode.getGatewayDeviceId(), gatewayNode);
+        if (result) {
+            updateGatewayLoadBalance(gatewayNode, false);
+        }
+        return result;
+    }
+
+    private void updateGatewayLoadBalance(GatewayNode gatewayNode, boolean nodeInsertion) {
+        deviceService.getAvailableDevices().forEach(device ->
+                groupService.getGroups(device.id(), appId).forEach(group ->
+                        selectGroupHandler.updateBucketToSelectGroupInVxlan(device.id(), group.appCookie(),
+                                Lists.newArrayList(gatewayNode), nodeInsertion)));
     }
     }
 
 
     private class InternalConfigListener implements NetworkConfigListener {
     private class InternalConfigListener implements NetworkConfigListener {
@@ -211,6 +248,17 @@ public class ScalableGatewayManager implements ScalableGatewayService {
         }
         }
     }
     }
 
 
+    private class InternalDeviceListener implements DeviceListener {
+
+        @Override
+        public void event(DeviceEvent deviceEvent) {
+            if (deviceEvent.type() == DeviceEvent.Type.DEVICE_SUSPENDED ||
+                    deviceEvent.type() == DeviceEvent.Type.DEVICE_REMOVED) {
+                deleteGatewayNode(getGatewayNode(deviceEvent.subject().id()));
+            }
+        }
+    }
+
     private void readConfiguration() {
     private void readConfiguration() {
         config = configService.getConfig(appId, GatewayNodeConfig.class);
         config = configService.getConfig(appId, GatewayNodeConfig.class);
         if (config == null) {
         if (config == null) {

+<!--
+  ~ Copyright 2016-present Open Networking Laboratory
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<blueprint xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0">
+
+    <command-bundle xmlns="http://karaf.apache.org/xmlns/shell/v1.1.0">
+        <command>
+            <action class="org.onosproject.scalablegateway.cli.ScalableGatewayListCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.scalablegateway.cli.ScalableGatewayAddCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.scalablegateway.cli.ScalableGatewayDeleteCommand"/>
+        </command>
+    </command-bundle>
+</blueprint>