【ONOS-1223】【ONOS-1870】the implements of label resource APIs.it include

commands
used to test
if there is any bug,LabelResourceManager,LabelResourceStore using
copycat,and junit test code.
the distribution strategy is that the master of devices handle all the
requests if applied label belongs to it.except for query request.
label store uses copycat instead of hazelcast to keep strong consistency

Change-Id: I77bde6a96f33098063573d37ed1ba787ae21973f

+package org.onosproject.cli.net;
+
+import java.util.Collection;
+import java.util.Iterator;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.resource.DefaultLabelResource;
+import org.onosproject.net.resource.LabelResource;
+import org.onosproject.net.resource.LabelResourceService;
+
+@Command(scope = "onos", name = "apply-global-label-resource-pool",
+      description = "Apply global labels from global resource pool")
+public class ApplyGlobalLabelResourceCommand extends AbstractShellCommand {
+    @Argument(index = 0, name = "applyNum",
+            description = "Applying number means how many labels applications want to use.",
+            required = true, multiValued = false)
+    String applyNum = null;
+
+    private static final String FMT = "deviceid=%s, labelresourceid=%s";
+
+    @Override
+    protected void execute() {
+        LabelResourceService lrs = get(LabelResourceService.class);
+        Collection<LabelResource> result =
+                lrs.applyFromGlobalPool(Long.parseLong(applyNum));
+        if (result.size() > 0) {
+            for (Iterator<LabelResource> iterator = result.iterator(); iterator
+                    .hasNext();) {
+                DefaultLabelResource defaultLabelResource = (DefaultLabelResource) iterator
+                        .next();
+                print(FMT, defaultLabelResource.deviceId().toString(),
+                      defaultLabelResource.labelResourceId().toString());
+            }
+        }
+    }
+
+}

+package org.onosproject.cli.net;
+
+import java.util.Collection;
+import java.util.Iterator;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.resource.DefaultLabelResource;
+import org.onosproject.net.resource.LabelResource;
+import org.onosproject.net.resource.LabelResourceService;
+
+@Command(scope = "onos", name = "apply-label-resource-pool",
+      description = "Apply label resource from device pool by specific device id")
+public class ApplyLabelResourceCommand extends AbstractShellCommand {
+    @Argument(index = 0, name = "deviceId",
+            description = "Device identity",
+            required = true, multiValued = false)
+    String deviceId = null;
+    @Argument(index = 1, name = "applyNum",
+            description = "Applying number means how many labels applications want to use.",
+            required = true, multiValued = false)
+    String applyNum = null;
+
+    private static final String FMT = "deviceid=%s, labelresourceid=%s";
+
+    @Override
+    protected void execute() {
+        LabelResourceService lrs = get(LabelResourceService.class);
+        Collection<LabelResource> result = lrs.applyFromDevicePool(DeviceId
+                .deviceId(deviceId), Long.parseLong(applyNum));
+        if (result.size() > 0) {
+            for (Iterator<LabelResource> iterator = result.iterator(); iterator
+                    .hasNext();) {
+                DefaultLabelResource defaultLabelResource = (DefaultLabelResource) iterator
+                        .next();
+                print(FMT, defaultLabelResource.deviceId().toString(),
+                      defaultLabelResource.labelResourceId().toString());
+            }
+        }
+    }
+
+}

+package org.onosproject.cli.net;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.resource.LabelResourceAdminService;
+import org.onosproject.net.resource.LabelResourceId;
+
+/**
+ * create label resource pool by specific device id.
+ */
+@Command(scope = "onos", name = "create-global-label-resource-pool",
+description = "Creates global label resource pool.")
+public class CreateGlobalLabelResourcePoolCommand extends AbstractShellCommand {
+    @Argument(index = 0, name = "beginLabel",
+            description = "The first label of global label resource pool.",
+            required = true, multiValued = false)
+    String beginLabel = null;
+    @Argument(index = 1, name = "endLabel",
+            description = "The last label of global label resource pool.",
+            required = true, multiValued = false)
+    String endLabel = null;
+
+    @Override
+    protected void execute() {
+        LabelResourceAdminService lrs = get(LabelResourceAdminService.class);
+        lrs.createGlobalPool(LabelResourceId.labelResourceId(Long
+                .parseLong(beginLabel)), LabelResourceId.labelResourceId(Long
+                .parseLong(endLabel)));
+    }
+
+}

+package org.onosproject.cli.net;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.resource.LabelResourceAdminService;
+import org.onosproject.net.resource.LabelResourceId;
+
+/**
+ * create label resource pool by specific device id.
+ */
+@Command(scope = "onos", name = "create-label-resource-pool",
+     description = "Creates label resource pool by a specific device id")
+public class CreateLabelResourcePoolCommand extends AbstractShellCommand {
+    @Argument(index = 0, name = "deviceId", description = "Device identity", required = true, multiValued = false)
+    String deviceId = null;
+    @Argument(index = 1, name = "beginLabel",
+            description = "The first label of global label resource pool.", required = true, multiValued = false)
+    String beginLabel = null;
+    @Argument(index = 2, name = "endLabel",
+            description = "The last label of global label resource pool.", required = true, multiValued = false)
+    String endLabel = null;
+
+    @Override
+    protected void execute() {
+        LabelResourceAdminService lrs = get(LabelResourceAdminService.class);
+        lrs.createDevicePool(DeviceId.deviceId(deviceId), LabelResourceId
+                .labelResourceId(Long.parseLong(beginLabel)), LabelResourceId
+                .labelResourceId(Long.parseLong(endLabel)));
+    }
+
+}

+package org.onosproject.cli.net;
+
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.resource.LabelResourceAdminService;
+
+@Command(scope = "onos", name = "destroy-global-label-resource-pool",
+description = "Destroys global label resource pool")
+public class DestroyGlobalLabelResourcePoolCommand extends AbstractShellCommand {
+    @Override
+    protected void execute() {
+        LabelResourceAdminService lrs = get(LabelResourceAdminService.class);
+        lrs.destroyGlobalPool();
+    }
+
+}

+package org.onosproject.cli.net;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.resource.LabelResourceAdminService;
+
+@Command(scope = "onos", name = "destroy-label-resource-pool",
+    description = "Destroys label resource pool by a specific device id")
+public class DestroyLabelResourcePoolCommand extends AbstractShellCommand {
+    @Argument(index = 0, name = "deviceId", description = "Device identity", required = true, multiValued = false)
+    String deviceId = null;
+
+    @Override
+    protected void execute() {
+        LabelResourceAdminService lrs = get(LabelResourceAdminService.class);
+        lrs.destroyDevicePool(DeviceId.deviceId(deviceId));
+    }
+
+}

+package org.onosproject.cli.net;
+
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.resource.LabelResourcePool;
+import org.onosproject.net.resource.LabelResourceService;
+
+@Command(scope = "onos", name = "get-global-label-resource-pool",
+      description = "Gets global label resource pool information.")
+public class GetGlobalLabelResourceCommand extends AbstractShellCommand {
+    private static final String FMT = "deviceid=%s, beginLabel=%s,"
+            + "endLabel=%s, totalNum=%s, usedNum=%s, currentUsedMaxLabelId=%s,"
+            + "releaseLabelIds=%s";
+
+    @Override
+    protected void execute() {
+        LabelResourceService lrs = get(LabelResourceService.class);
+        LabelResourcePool pool = lrs.getGlobalLabelResourcePool();
+        if (pool != null) {
+            print(FMT, pool.deviceId().toString(), pool.beginLabel(),
+                  pool.endLabel(), pool.totalNum(), pool.usedNum(),
+                  pool.currentUsedMaxLabelId(), pool.releaseLabelId()
+                          .toString());
+        }
+    }
+
+}

+package org.onosproject.cli.net;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.resource.LabelResourcePool;
+import org.onosproject.net.resource.LabelResourceService;
+
+@Command(scope = "onos", name = "get-label-resource-pool",
+      description = "Gets label resource pool information by a specific device id")
+public class GetLabelResourceCommand extends AbstractShellCommand {
+    @Argument(index = 0, name = "deviceId",
+            description = "Device identity", required = true, multiValued = false)
+    String deviceId = null;
+    private static final String FMT = "deviceid=%s, beginLabel=%s,"
+            + "endLabel=%s, totalNum=%s, usedNum=%s, currentUsedMaxLabelId=%s,"
+            + "releaseLabelIds=%s";
+
+    @Override
+    protected void execute() {
+        LabelResourceService lrs = get(LabelResourceService.class);
+        LabelResourcePool pool = lrs.getDeviceLabelResourcePool(DeviceId
+                .deviceId(deviceId));
+        if (pool != null) {
+            print(FMT, pool.deviceId().toString(), pool.beginLabel(),
+                  pool.endLabel(), pool.totalNum(), pool.usedNum(),
+                  pool.currentUsedMaxLabelId(), pool.releaseLabelId()
+                          .toString());
+        } else {
+            print(FMT, deviceId, null, null, null, null, null, null);
+        }
+    }
+
+}

+package org.onosproject.cli.net;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.resource.LabelResourceId;
+import org.onosproject.net.resource.LabelResourceService;
+
+@Command(scope = "onos", name = "release-global-label-resource-pool",
+description = "Releases labels to global label resource pool.")
+public class ReleaseGlobalLabelResourceCommand extends AbstractShellCommand {
+    @Argument(index = 0, name = "releaseLabelIds",
+            description = "Represents for the label ids that are released. They are splited by dot symbol",
+            required = true, multiValued = false)
+    String releaseLabelIds = null;
+
+    @Override
+    protected void execute() {
+        LabelResourceService lrs = get(LabelResourceService.class);
+        Set<LabelResourceId> release = new HashSet<LabelResourceId>();
+        String[] labelIds = releaseLabelIds.split(",");
+        LabelResourceId resource = null;
+        for (int i = 0; i < labelIds.length; i++) {
+            resource = LabelResourceId.labelResourceId(Long.parseLong(labelIds[i]));
+            release.add(resource);
+        }
+        lrs.releaseToGlobalPool(release);
+    }
+
+}

+package org.onosproject.cli.net;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.resource.DefaultLabelResource;
+import org.onosproject.net.resource.LabelResource;
+import org.onosproject.net.resource.LabelResourceId;
+import org.onosproject.net.resource.LabelResourceService;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+
+@Command(scope = "onos", name = "release-label-resource-pool",
+description = "Releases label ids to label resource pool by a specific device id")
+public class ReleaseLabelResourceCommand extends AbstractShellCommand {
+    @Argument(index = 0, name = "deviceId",
+            description = "Device identity",
+            required = true, multiValued = false)
+    String deviceId = null;
+    @Argument(index = 1, name = "releaseLabelIds",
+            description = "Represents for the label ids that are released. They are splited by dot symbol",
+            required = true, multiValued = false)
+    String releaseLabelIds = null;
+
+    @Override
+    protected void execute() {
+        LabelResourceService lrs = get(LabelResourceService.class);
+        Multimap<DeviceId, LabelResource> map = ArrayListMultimap
+                .create();
+        String[] labelIds = releaseLabelIds.split(",");
+        DefaultLabelResource resource = null;
+        for (int i = 0; i < labelIds.length; i++) {
+            resource = new DefaultLabelResource(
+                                                DeviceId.deviceId(deviceId),
+                                                LabelResourceId.labelResourceId(Long
+                                                        .parseLong(labelIds[i])));
+            map.put(DeviceId.deviceId(deviceId), resource);
+        }
+        lrs.releaseToDevicePool(map);
+    }
+
+}

@@ -38,7 +38,6 @@
                 <ref component-id="cfgCommandCompleter"/>
                 <ref component-id="componentNameCompleter"/>
                 <ref component-id="componentPropertyNameCompleter"/>
-                <null/>
             </completers>
         </command>
 
@@ -67,7 +66,6 @@
             <action class="org.onosproject.cli.net.DriversListCommand"/>
             <completers>
                 <ref component-id="driverNameCompleter"/>
-                <null/>
             </completers>
         </command>
 
@@ -78,14 +76,12 @@
             <action class="org.onosproject.cli.net.DevicePortsListCommand"/>
             <completers>
                 <ref component-id="deviceIdCompleter"/>
-                <null/>
             </completers>
         </command>
         <command>
             <action class="org.onosproject.cli.net.DeviceRemoveCommand"/>
             <completers>
                 <ref component-id="deviceIdCompleter"/>
-                <null/>
             </completers>
         </command>
         <command>
@@ -94,14 +90,12 @@
                 <ref component-id="deviceIdCompleter"/>
                 <ref component-id="nodeIdCompleter"/>
                 <ref component-id="roleCompleter"/>
-                <null/>
             </completers>
         </command>
         <command>
             <action class="org.onosproject.cli.net.AnnotateDeviceCommand"/>
             <completers>
                 <ref component-id="deviceIdCompleter"/>
-                <null/>
             </completers>
         </command>
 
@@ -109,7 +103,6 @@
             <action class="org.onosproject.cli.net.LinksListCommand"/>
             <completers>
                 <ref component-id="deviceIdCompleter"/>
-                <null/>
             </completers>
         </command>
 
@@ -121,7 +114,6 @@
             <completers>
                 <ref component-id="deviceIdCompleter"/>
                 <ref component-id="deviceIdCompleter"/>
-                <null/>
             </completers>
         </command>
 
@@ -133,7 +125,6 @@
             <completers>
                 <ref component-id="appIdWithIntentNameCompleter"/>
                 <ref component-id="intentIdCompleter"/>
-                <null/>
             </completers>
         </command>
         <command>
@@ -141,7 +132,6 @@
             <completers>
                 <ref component-id="hostIdCompleter"/>
                 <ref component-id="hostIdCompleter"/>
-                <null/>
             </completers>
             <optional-completers>
                 <entry key="-t" value-ref="ethTypeCompleter"/>
@@ -157,7 +147,6 @@
             <completers>
                 <ref component-id="connectPointCompleter"/>
                 <ref component-id="connectPointCompleter"/>
-                <null/>
             </completers>
             <optional-completers>
                 <entry key="-t" value-ref="ethTypeCompleter"/>
@@ -173,7 +162,6 @@
             <completers>
                 <ref component-id="connectPointCompleter"/>
                 <ref component-id="connectPointCompleter"/>
-                <null/>
             </completers>
             <optional-completers>
                 <entry key="-a" value-ref="allAppNameCompleter"/>
@@ -219,7 +207,6 @@
                 <ref component-id="connectPointCompleter"/>
                 <ref component-id="connectPointCompleter"/>
                 <ref component-id="nullCompleter"/>
-                <null/>
             </completers>
         </command>
         <command>
@@ -228,14 +215,12 @@
                 <ref component-id="connectPointCompleter"/>
                 <ref component-id="connectPointCompleter"/>
                 <ref component-id="nullCompleter"/>
-                <null/>
             </completers>
         </command>
         <command>
             <action class="org.onosproject.cli.net.RandomIntentCommand"/>
             <completers>
                 <ref component-id="nullCompleter"/>
-                <null/>
             </completers>
         </command>
         <command>
@@ -243,7 +228,6 @@
             <completers>
                 <ref component-id="connectPointCompleter"/>
                 <ref component-id="connectPointCompleter"/>
-                <null/>
             </completers>
         </command>
         <command>
@@ -251,7 +235,6 @@
             <completers>
                 <ref component-id="connectPointCompleter"/>
                 <ref component-id="connectPointCompleter"/>
-                <null/>
             </completers>
         </command>
         <command>
@@ -281,14 +264,12 @@
             <action class="org.onosproject.cli.net.ClusterDevicesCommand"/>
             <completers>
                 <ref component-id="clusterIdCompleter"/>
-                <null/>
             </completers>
         </command>
         <command>
             <action class="org.onosproject.cli.net.ClusterLinksCommand"/>
             <completers>
                 <ref component-id="clusterIdCompleter"/>
-                <null/>
             </completers>
         </command>
 
@@ -299,7 +280,6 @@
             <action class="org.onosproject.cli.net.HostRemoveCommand"/>
             <completers>
                 <ref component-id="hostIdCompleter"/>
-                <null/>
             </completers>
         </command>
         <command>
@@ -319,7 +299,6 @@
             <completers>
                 <ref component-id="flowRuleStatusCompleter"/>
                 <ref component-id="deviceIdCompleter"/>
-                <null/>
             </completers>
         </command>
 
@@ -338,7 +317,6 @@
             <completers>
                 <ref component-id="connectPointCompleter"/>
                 <ref component-id="connectPointCompleter"/>
-                <null/>
             </completers>
             <optional-completers>
                 <entry key="-t" value-ref="ethTypeCompleter"/>
@@ -349,6 +327,37 @@
                 <entry key="-a" value-ref="allAppNameCompleter"/>
             </optional-completers>
         </command>
+        
+        <command>
+            <action class="org.onosproject.cli.net.GetGlobalLabelResourceCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.cli.net.GetLabelResourceCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.cli.net.CreateGlobalLabelResourcePoolCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.cli.net.CreateLabelResourcePoolCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.cli.net.DestroyGlobalLabelResourcePoolCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.cli.net.DestroyGlobalLabelResourcePoolCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.cli.net.ReleaseGlobalLabelResourceCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.cli.net.ReleaseLabelResourceCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.cli.net.ApplyGlobalLabelResourceCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.cli.net.ApplyLabelResourceCommand"/>
+        </command>
     </command-bundle>
 
     <bean id="appCommandCompleter" class="org.onosproject.cli.app.ApplicationCommandCompleter"/>

+package org.onosproject.net.resource;
+
+import org.junit.Test;
+import org.onosproject.event.AbstractEventTest;
+
+import com.google.common.testing.EqualsTester;
+
+/**
+ * Tests of default label resource.
+ */
+public class DefaultLabelResourceTest extends AbstractEventTest {
+
+    @Test
+    public void testEquality() {
+        String deviceId1 = "of:001";
+        String deviceId2 = "of:002";
+        long labelResourceId1 = 100;
+        long labelResourceId2 = 200;
+        DefaultLabelResource h1 = new DefaultLabelResource(deviceId1,
+                                                           labelResourceId1);
+        DefaultLabelResource h2 = new DefaultLabelResource(deviceId1,
+                                                           labelResourceId1);
+        DefaultLabelResource h3 = new DefaultLabelResource(deviceId2,
+                                                           labelResourceId2);
+        DefaultLabelResource h4 = new DefaultLabelResource(deviceId2,
+                                                           labelResourceId2);
+
+        new EqualsTester().addEqualityGroup(h1, h2).addEqualityGroup(h3, h4)
+                .testEquals();
+    }
+}

+package org.onosproject.net.resource;
+
+import org.junit.Test;
+import org.onosproject.event.AbstractEventTest;
+
+import com.google.common.testing.EqualsTester;
+
+/**
+ * Tests of the label resource pool.
+ */
+public class LabelResourcePoolTest extends AbstractEventTest {
+
+    @Test
+    public void testEquality() {
+        LabelResourcePool h1 = new LabelResourcePool("of:001", 0, 100);
+        LabelResourcePool h2 = new LabelResourcePool("of:001", 0, 100);
+        LabelResourcePool h3 = new LabelResourcePool("of:002", 0, 100);
+        LabelResourcePool h4 = new LabelResourcePool("of:002", 0, 100);
+        new EqualsTester().addEqualityGroup(h1, h2).addEqualityGroup(h3, h4)
+                .testEquals();
+    }
+
+}

+package org.onosproject.net.resource;
+
+import java.util.Collections;
+
+import org.junit.Test;
+import org.onosproject.event.AbstractEventTest;
+import org.onosproject.net.DeviceId;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.testing.EqualsTester;
+
+/**
+ * Tests of the label resource request.
+ */
+public class LabelResourceRequestTest extends AbstractEventTest {
+
+    @Test
+    public void testEquality() {
+        DeviceId deviceId1 = DeviceId.deviceId("of:0001");
+        DeviceId deviceId2 = DeviceId.deviceId("of:0002");
+        long apply = 2;
+        ImmutableSet<LabelResource> releaseCollection = ImmutableSet
+                .copyOf(Collections.emptySet());
+        LabelResourceRequest h1 = new LabelResourceRequest(
+                                                           deviceId1,
+                                                           LabelResourceRequest.Type.APPLY,
+                                                           apply, null);
+        LabelResourceRequest h2 = new LabelResourceRequest(
+                                                           deviceId1,
+                                                           LabelResourceRequest.Type.APPLY,
+                                                           apply, null);
+        LabelResourceRequest h3 = new LabelResourceRequest(
+                                                           deviceId2,
+                                                           LabelResourceRequest.Type.RELEASE,
+                                                           0, releaseCollection);
+        LabelResourceRequest h4 = new LabelResourceRequest(
+                                                           deviceId2,
+                                                           LabelResourceRequest.Type.RELEASE,
+                                                           0, releaseCollection);
+
+        new EqualsTester().addEqualityGroup(h1, h2).addEqualityGroup(h3, h4)
+                .testEquals();
+    }
+}

+package org.onosproject.net.resource.impl;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onosproject.event.AbstractListenerRegistry;
+import org.onosproject.event.EventDeliveryService;
+import org.onosproject.net.Device;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.device.DeviceEvent;
+import org.onosproject.net.device.DeviceEvent.Type;
+import org.onosproject.net.device.DeviceListener;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.provider.AbstractProviderRegistry;
+import org.onosproject.net.provider.AbstractProviderService;
+import org.onosproject.net.resource.LabelResource;
+import org.onosproject.net.resource.LabelResourceAdminService;
+import org.onosproject.net.resource.LabelResourceDelegate;
+import org.onosproject.net.resource.LabelResourceEvent;
+import org.onosproject.net.resource.LabelResourceId;
+import org.onosproject.net.resource.LabelResourceListener;
+import org.onosproject.net.resource.LabelResourcePool;
+import org.onosproject.net.resource.LabelResourceProvider;
+import org.onosproject.net.resource.LabelResourceProviderRegistry;
+import org.onosproject.net.resource.LabelResourceProviderService;
+import org.onosproject.net.resource.LabelResourceService;
+import org.onosproject.net.resource.LabelResourceStore;
+import org.slf4j.Logger;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.Multimap;
+
+/**
+ * provides implementation of the label resource NB & SB APIs.
+ *
+ */
+@Component(immediate = true)
+@Service
+public class LabelResourceManager
+        extends
+        AbstractProviderRegistry<LabelResourceProvider, LabelResourceProviderService>
+        implements LabelResourceService, LabelResourceAdminService,
+        LabelResourceProviderRegistry {
+    private final Logger log = getLogger(getClass());
+    private final LabelResourceDelegate delegate = new InternalLabelResourceDelegate();
+
+    private final AbstractListenerRegistry<LabelResourceEvent, LabelResourceListener> listenerRegistry
+                            = new AbstractListenerRegistry<>();
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected LabelResourceStore store;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected EventDeliveryService eventDispatcher;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DeviceService deviceService;
+
+    private DeviceListener deviceListener = new InternalDeviceListener();
+
+    @Activate
+    public void activate() {
+        store.setDelegate(delegate);
+        eventDispatcher.addSink(LabelResourceEvent.class, listenerRegistry);
+        deviceService.addListener(deviceListener);
+        log.info("Started");
+
+    }
+
+    @Deactivate
+    public void deactivate() {
+        deviceService.removeListener(deviceListener);
+        store.unsetDelegate(delegate);
+        eventDispatcher.removeSink(LabelResourceEvent.class);
+        log.info("Stopped");
+    }
+
+    @Override
+    public boolean createDevicePool(DeviceId deviceId,
+                                    LabelResourceId beginLabel,
+                                    LabelResourceId endLabel) {
+        checkNotNull(deviceId, "deviceId is not null");
+        checkNotNull(beginLabel, "beginLabel is not null");
+        checkNotNull(endLabel, "beginLabel is not null");
+        checkArgument(beginLabel.labelId() < 0 || endLabel.labelId() < 0,
+                      "The value of beginLabel and the value of endLabel must be both positive number.");
+        checkArgument(beginLabel.labelId() > endLabel.labelId(),
+                      "The value of endLabel must be greater than the value of endLabel.");
+        return store.createDevicePool(deviceId, beginLabel, endLabel);
+    }
+
+    @Override
+    public boolean createGlobalPool(LabelResourceId beginLabel,
+                                    LabelResourceId endLabel) {
+        checkNotNull(beginLabel, "beginLabel is not null");
+        checkNotNull(endLabel, "beginLabel is not null");
+        checkArgument(beginLabel.labelId() < 0 || endLabel.labelId() < 0,
+                      "The value of beginLabel and the value of endLabel must be both positive number.");
+        checkArgument(beginLabel.labelId() > endLabel.labelId(),
+                      "The value of endLabel must be greater than the value of endLabel.");
+        return store.createGlobalPool(beginLabel, endLabel);
+    }
+
+    @Override
+    public boolean destroyDevicePool(DeviceId deviceId) {
+        checkNotNull(deviceId, "deviceId is not null");
+        return store.destroyDevicePool(deviceId);
+    }
+
+    @Override
+    public boolean destroyGlobalPool() {
+        return store.destroyGlobalPool();
+    }
+
+    @Override
+    public Collection<LabelResource> applyFromDevicePool(DeviceId deviceId,
+                                                         long applyNum) {
+        checkNotNull(deviceId, "deviceId is not null");
+        checkNotNull(applyNum, "applyNum is not null");
+        return store.applyFromDevicePool(deviceId, applyNum);
+    }
+
+    @Override
+    public Collection<LabelResource> applyFromGlobalPool(long applyNum) {
+        checkNotNull(applyNum, "applyNum is not null");
+        return store.applyFromGlobalPool(applyNum);
+    }
+
+    @Override
+    public boolean releaseToDevicePool(Multimap<DeviceId, LabelResource> release) {
+        checkNotNull(release, "release is not null");
+        return store.releaseToDevicePool(release);
+    }
+
+    @Override
+    public boolean releaseToGlobalPool(Set<LabelResourceId> release) {
+        checkNotNull(release, "release is not null");
+        return store.releaseToGlobalPool(release);
+    }
+
+    @Override
+    public boolean isDevicePoolFull(DeviceId deviceId) {
+        checkNotNull(deviceId, "deviceId is not null");
+        return store.isDevicePoolFull(deviceId);
+    }
+
+    @Override
+    public boolean isGlobalPoolFull() {
+        return store.isGlobalPoolFull();
+    }
+
+    @Override
+    public long getFreeNumOfDevicePool(DeviceId deviceId) {
+        checkNotNull(deviceId, "deviceId is not null");
+        return store.getFreeNumOfDevicePool(deviceId);
+    }
+
+    @Override
+    public long getFreeNumOfGlobalPool() {
+        return store.getFreeNumOfGlobalPool();
+    }
+
+    @Override
+    public LabelResourcePool getDeviceLabelResourcePool(DeviceId deviceId) {
+        checkNotNull(deviceId, "deviceId is not null");
+        return store.getDeviceLabelResourcePool(deviceId);
+    }
+
+    @Override
+    public LabelResourcePool getGlobalLabelResourcePool() {
+        return store.getGlobalLabelResourcePool();
+    }
+
+    @Override
+    public void addListener(LabelResourceListener listener) {
+        listenerRegistry.addListener(listener);
+    }
+
+    @Override
+    public void removeListener(LabelResourceListener listener) {
+        listenerRegistry.removeListener(listener);
+
+    }
+
+    private void post(LabelResourceEvent event) {
+        if (event != null) {
+            eventDispatcher.post(event);
+        }
+    }
+
+    private class InternalLabelResourceDelegate
+            implements LabelResourceDelegate {
+
+        @Override
+        public void notify(LabelResourceEvent event) {
+            post(event);
+        }
+
+    }
+
+    private class InternalDeviceListener implements DeviceListener {
+
+        @Override
+        public void event(DeviceEvent event) {
+            Device device = event.subject();
+            if (Type.DEVICE_REMOVED.equals(event.type())) {
+                destroyDevicePool(device.id());
+            }
+        }
+    }
+
+    private class InternalLabelResourceProviderService
+            extends AbstractProviderService<LabelResourceProvider>
+            implements LabelResourceProviderService {
+
+        protected InternalLabelResourceProviderService(LabelResourceProvider provider) {
+            super(provider);
+        }
+
+        @Override
+        public void deviceLabelResourcePoolDetected(DeviceId deviceId,
+                                                    LabelResourceId beginLabel,
+                                                    LabelResourceId endLabel) {
+            checkNotNull(deviceId, "deviceId is not null");
+            checkNotNull(beginLabel, "beginLabel is not null");
+            checkNotNull(endLabel, "endLabel is not null");
+            createDevicePool(deviceId, beginLabel, endLabel);
+        }
+
+        @Override
+        public void deviceLabelResourcePoolDestroyed(DeviceId deviceId) {
+            checkNotNull(deviceId, "deviceId is not null");
+            destroyDevicePool(deviceId);
+        }
+
+    }
+
+    @Override
+    protected LabelResourceProviderService createProviderService(LabelResourceProvider provider) {
+        return new InternalLabelResourceProviderService(provider);
+    }
+}

+package org.onosproject.store.resource.impl;
+
+import static org.onlab.util.Tools.groupedThreads;
+import static org.slf4j.LoggerFactory.getLogger;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.net.Device;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.resource.DefaultLabelResource;
+import org.onosproject.net.resource.LabelResource;
+import org.onosproject.net.resource.LabelResourceDelegate;
+import org.onosproject.net.resource.LabelResourceEvent;
+import org.onosproject.net.resource.LabelResourceEvent.Type;
+import org.onosproject.net.resource.LabelResourceId;
+import org.onosproject.net.resource.LabelResourcePool;
+import org.onosproject.net.resource.LabelResourceRequest;
+import org.onosproject.net.resource.LabelResourceStore;
+import org.onosproject.store.AbstractStore;
+import org.onosproject.store.cluster.messaging.ClusterCommunicationService;
+import org.onosproject.store.cluster.messaging.ClusterMessage;
+import org.onosproject.store.cluster.messaging.ClusterMessageHandler;
+import org.onosproject.store.flow.ReplicaInfo;
+import org.onosproject.store.flow.ReplicaInfoService;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.serializers.KryoSerializer;
+import org.onosproject.store.serializers.impl.DistributedStoreSerializers;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
+import org.slf4j.Logger;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+
+/**
+ * Manages label resources using copycat.
+ */
+@Component(immediate = true, enabled = true)
+@Service
+public class DistributedLabelResourceStore
+        extends AbstractStore<LabelResourceEvent, LabelResourceDelegate>
+        implements LabelResourceStore {
+    private final Logger log = getLogger(getClass());
+
+    private static final String POOL_MAP_NAME = "labelresourcepool";
+
+    private static final String GLOBAL_RESOURCE_POOL_DEVICE_ID = "global_resource_pool_device_id";
+    // primary data:
+    // read/write needs to be locked
+    private final ReentrantReadWriteLock resourcePoolLock = new ReentrantReadWriteLock();
+
+    private ConsistentMap<DeviceId, LabelResourcePool> resourcePool = null;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected StorageService storageService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ReplicaInfoService replicaInfoManager;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterCommunicationService clusterCommunicator;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterService clusterService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DeviceService deviceService;
+
+    private ExecutorService messageHandlingExecutor;
+    private static final int MESSAGE_HANDLER_THREAD_POOL_SIZE = 8;
+    private static final long PEER_REQUEST_TIMEOUT_MS = 5000;
+
+    protected static final KryoSerializer SERIALIZER = new KryoSerializer() {
+        @Override
+        protected void setupKryoPool() {
+            serializerPool = KryoNamespace.newBuilder()
+                    .register(DistributedStoreSerializers.STORE_COMMON)
+                    .nextId(DistributedStoreSerializers.STORE_CUSTOM_BEGIN)
+                    .register(LabelResourceEvent.class)
+                    .register(LabelResourcePool.class).register(DeviceId.class)
+                    .register(LabelResourceRequest.class)
+                    .register(LabelResourceRequest.Type.class)
+                    .register(LabelResourceEvent.Type.class)
+                    .register(DefaultLabelResource.class)
+                    .register(LabelResourceId.class).build();
+        }
+    };
+
+    @Activate
+    public void activate() {
+
+        resourcePool = storageService
+                .<DeviceId, LabelResourcePool>consistentMapBuilder()
+                .withName(POOL_MAP_NAME).withSerializer(new Serializer() {
+                    KryoNamespace kryo = new KryoNamespace.Builder()
+                            .register(KryoNamespaces.API).build();
+
+                    @Override
+                    public <T> byte[] encode(T object) {
+                        return kryo.serialize(object);
+                    }
+
+                    @Override
+                    public <T> T decode(byte[] bytes) {
+                        return kryo.deserialize(bytes);
+                    }
+                }).withPartitionsDisabled().build();
+        messageHandlingExecutor = Executors
+                .newFixedThreadPool(MESSAGE_HANDLER_THREAD_POOL_SIZE,
+                                    groupedThreads("onos/store/flow",
+                                                   "message-handlers"));
+        clusterCommunicator
+                .addSubscriber(LabelResourceMessageSubjects.LABEL_POOL_CREATED,
+                               new ClusterMessageHandler() {
+
+                                   @Override
+                                   public void handle(ClusterMessage message) {
+                                       LabelResourcePool operation = SERIALIZER
+                                               .decode(message.payload());
+                                       log.trace("received get flow entry request for {}",
+                                                 operation);
+                                       boolean b = internalCreate(operation);
+                                           message.respond(SERIALIZER.encode(b));
+                                   }
+                               }, messageHandlingExecutor);
+        clusterCommunicator
+                .addSubscriber(LabelResourceMessageSubjects.LABEL_POOL_DESTROYED,
+                               new ClusterMessageHandler() {
+
+                                   @Override
+                                   public void handle(ClusterMessage message) {
+                                       DeviceId deviceId = SERIALIZER
+                                               .decode(message.payload());
+                                       log.trace("received get flow entry request for {}",
+                                                 deviceId);
+                                       boolean b = internalDestroy(deviceId);
+                                           message.respond(SERIALIZER.encode(b));
+                                   }
+                               }, messageHandlingExecutor);
+        clusterCommunicator
+                .addSubscriber(LabelResourceMessageSubjects.LABEL_POOL_APPLY,
+                               new ClusterMessageHandler() {
+
+                                   @Override
+                                   public void handle(ClusterMessage message) {
+                                       LabelResourceRequest request = SERIALIZER
+                                               .decode(message.payload());
+                                       log.trace("received get flow entry request for {}",
+                                                 request);
+                                       final Collection<LabelResource> resource = internalApply(request);
+                                           message.respond(SERIALIZER
+                                                   .encode(resource));
+                                   }
+                               }, messageHandlingExecutor);
+        clusterCommunicator
+                .addSubscriber(LabelResourceMessageSubjects.LABEL_POOL_RELEASE,
+                               new ClusterMessageHandler() {
+
+                                   @Override
+                                   public void handle(ClusterMessage message) {
+                                       LabelResourceRequest request = SERIALIZER
+                                               .decode(message.payload());
+                                       log.trace("received get flow entry request for {}",
+                                                 request);
+                                       final boolean isSuccess = internalRelease(request);
+                                           message.respond(SERIALIZER
+                                                   .encode(isSuccess));
+                                   }
+                               }, messageHandlingExecutor);
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        clusterCommunicator
+                .removeSubscriber(LabelResourceMessageSubjects.LABEL_POOL_CREATED);
+        clusterCommunicator
+                .removeSubscriber(LabelResourceMessageSubjects.LABEL_POOL_APPLY);
+        clusterCommunicator
+                .removeSubscriber(LabelResourceMessageSubjects.LABEL_POOL_DESTROYED);
+        clusterCommunicator
+                .removeSubscriber(LabelResourceMessageSubjects.LABEL_POOL_RELEASE);
+        messageHandlingExecutor.shutdown();
+        log.info("Stopped");
+    }
+
+    @Override
+    public boolean createDevicePool(DeviceId deviceId,
+                                    LabelResourceId beginLabel,
+                                    LabelResourceId endLabel) {
+        LabelResourcePool pool = new LabelResourcePool(deviceId.toString(),
+                                                       beginLabel.labelId(),
+                                                       endLabel.labelId());
+        return this.create(pool);
+    }
+
+    @Override
+    public boolean createGlobalPool(LabelResourceId beginLabel,
+                                    LabelResourceId endLabel) {
+        LabelResourcePool pool = new LabelResourcePool(
+                                                       GLOBAL_RESOURCE_POOL_DEVICE_ID,
+                                                       beginLabel.labelId(),
+                                                       endLabel.labelId());
+        return this.internalCreate(pool);
+    }
+
+    private boolean create(LabelResourcePool pool) {
+        Device device = (Device) deviceService.getDevice(pool.deviceId());
+        if (device == null) {
+            return false;
+        }
+
+        ReplicaInfo replicaInfo = replicaInfoManager.getReplicaInfoFor(pool
+                .deviceId());
+
+        if (!replicaInfo.master().isPresent()) {
+            log.warn("Failed to getFlowEntries: No master for {}", pool);
+            return false;
+        }
+
+        if (replicaInfo.master().get()
+                .equals(clusterService.getLocalNode().id())) {
+            return internalCreate(pool);
+        }
+
+        log.trace("Forwarding getFlowEntries to {}, which is the primary (master) for device {}",
+                  replicaInfo.master().orNull(), pool.deviceId());
+
+        return complete(clusterCommunicator
+                .sendAndReceive(pool,
+                                LabelResourceMessageSubjects.LABEL_POOL_CREATED,
+                                SERIALIZER::encode, SERIALIZER::decode,
+                                replicaInfo.master().get()));
+    }
+
+    private boolean internalCreate(LabelResourcePool pool) {
+        resourcePoolLock.writeLock().lock();
+        LabelResourcePool poolOld = resourcePool.get(pool.deviceId()).value();
+        if (poolOld == null) {
+            resourcePool.put(pool.deviceId(), pool);
+            resourcePoolLock.writeLock().unlock();
+            LabelResourceEvent event = new LabelResourceEvent(
+                                                              Type.POOL_CREATED,
+                                                              pool);
+            notifyDelegate(event);
+            return true;
+        }
+        resourcePoolLock.writeLock().unlock();
+        return false;
+    }
+
+    @Override
+    public boolean destroyDevicePool(DeviceId deviceId) {
+        Device device = (Device) deviceService.getDevice(deviceId);
+        if (device == null) {
+            return false;
+        }
+        ReplicaInfo replicaInfo = replicaInfoManager
+                .getReplicaInfoFor(deviceId);
+
+        if (!replicaInfo.master().isPresent()) {
+            log.warn("Failed to getFlowEntries: No master for {}", deviceId);
+            return false;
+        }
+
+        if (replicaInfo.master().get()
+                .equals(clusterService.getLocalNode().id())) {
+            return internalDestroy(deviceId);
+        }
+
+        log.trace("Forwarding getFlowEntries to {}, which is the primary (master) for device {}",
+                  replicaInfo.master().orNull(), deviceId);
+
+        return complete(clusterCommunicator
+                .sendAndReceive(deviceId,
+                                LabelResourceMessageSubjects.LABEL_POOL_DESTROYED,
+                                SERIALIZER::encode, SERIALIZER::decode,
+                                replicaInfo.master().get()));
+    }
+
+    private boolean internalDestroy(DeviceId deviceId) {
+        LabelResourcePool poolOld = resourcePool.get(deviceId).value();
+        if (poolOld != null) {
+            resourcePool.remove(deviceId);
+            LabelResourceEvent event = new LabelResourceEvent(
+                                                              Type.POOL_CREATED,
+                                                              poolOld);
+            notifyDelegate(event);
+        }
+        log.info("success to destroy the label resource pool of device id {}",
+                 deviceId);
+        return true;
+    }
+
+    @Override
+    public Collection<LabelResource> applyFromDevicePool(DeviceId deviceId,
+                                                         long applyNum) {
+        Device device = (Device) deviceService.getDevice(deviceId);
+        if (device == null) {
+            return Collections.emptyList();
+        }
+        LabelResourceRequest request = new LabelResourceRequest(
+                                                                deviceId,
+                                                                LabelResourceRequest.Type.APPLY,
+                                                                applyNum, null);
+        ReplicaInfo replicaInfo = replicaInfoManager
+                .getReplicaInfoFor(deviceId);
+
+        if (!replicaInfo.master().isPresent()) {
+            log.warn("Failed to getFlowEntries: No master for {}", deviceId);
+            return Collections.emptyList();
+        }
+
+        if (replicaInfo.master().get()
+                .equals(clusterService.getLocalNode().id())) {
+            return internalApply(request);
+        }
+
+        log.trace("Forwarding getFlowEntries to {}, which is the primary (master) for device {}",
+                  replicaInfo.master().orNull(), deviceId);
+
+        return complete(clusterCommunicator
+                .sendAndReceive(request,
+                                LabelResourceMessageSubjects.LABEL_POOL_APPLY,
+                                SERIALIZER::encode, SERIALIZER::decode,
+                                replicaInfo.master().get()));
+    }
+
+    private Collection<LabelResource> internalApply(LabelResourceRequest request) {
+        resourcePoolLock.writeLock().lock();
+        DeviceId deviceId = request.deviceId();
+        long applyNum = request.applyNum();
+        LabelResourcePool pool = resourcePool.get(deviceId).value();
+        Collection<LabelResource> result = new HashSet<LabelResource>();
+        long freeNum = this.getFreeNumOfDevicePool(deviceId);
+        if (applyNum > freeNum) {
+            log.info("the free number of the label resource pool of deviceId {} is not enough.");
+            resourcePoolLock.writeLock().unlock();
+            return Collections.emptyList();
+        }
+        Set<LabelResource> releaseLabels = new HashSet<LabelResource>(
+                                                                      pool.releaseLabelId());
+        long tmp = releaseLabels.size() > applyNum ? applyNum : releaseLabels
+                .size();
+        LabelResource resource = null;
+        for (int i = 0; i < tmp; i++) {
+            Iterator<LabelResource> it = releaseLabels.iterator();
+            if (it.hasNext()) {
+                resource = it.next();
+                releaseLabels.remove(resource);
+            }
+            result.add(resource);
+        }
+        for (long j = pool.currentUsedMaxLabelId().labelId(); j < pool
+                .currentUsedMaxLabelId().labelId() + applyNum - tmp; j++) {
+            resource = new DefaultLabelResource(deviceId,
+                                                LabelResourceId
+                                                        .labelResourceId(j));
+            result.add(resource);
+        }
+        long beginLabel = pool.beginLabel().labelId();
+        long endLabel = pool.endLabel().labelId();
+        long totalNum = pool.totalNum();
+        long current = pool.currentUsedMaxLabelId().labelId() + applyNum - tmp;
+        long usedNum = pool.usedNum() + applyNum;
+        ImmutableSet<LabelResource> freeLabel = ImmutableSet
+                .copyOf(releaseLabels);
+        LabelResourcePool newPool = new LabelResourcePool(deviceId.toString(),
+                                                          beginLabel, endLabel,
+                                                          totalNum, usedNum,
+                                                          current, freeLabel);
+        resourcePool.put(deviceId, newPool);
+        log.info("success to apply label resource");
+        resourcePoolLock.writeLock().unlock();
+        return result;
+    }
+
+    @Override
+    public boolean releaseToDevicePool(Multimap<DeviceId, LabelResource> release) {
+        Map<DeviceId, Collection<LabelResource>> maps = release.asMap();
+        Set<DeviceId> deviceIdSet = maps.keySet();
+        LabelResourceRequest request = null;
+        for (Iterator<DeviceId> it = deviceIdSet.iterator(); it.hasNext();) {
+            DeviceId deviceId = (DeviceId) it.next();
+            Device device = (Device) deviceService.getDevice(deviceId);
+            if (device == null) {
+                continue;
+            }
+            ImmutableSet<LabelResource> collection = ImmutableSet.copyOf(maps
+                    .get(deviceId));
+            request = new LabelResourceRequest(
+                                               deviceId,
+                                               LabelResourceRequest.Type.RELEASE,
+                                               0, collection);
+            ReplicaInfo replicaInfo = replicaInfoManager
+                    .getReplicaInfoFor(deviceId);
+
+            if (!replicaInfo.master().isPresent()) {
+                log.warn("Failed to getFlowEntries: No master for {}", deviceId);
+                return false;
+            }
+
+            if (replicaInfo.master().get()
+                    .equals(clusterService.getLocalNode().id())) {
+                return internalRelease(request);
+            }
+
+            log.trace("Forwarding getFlowEntries to {}, which is the primary (master) for device {}",
+                      replicaInfo.master().orNull(), deviceId);
+
+            return complete(clusterCommunicator
+                    .sendAndReceive(request,
+                                    LabelResourceMessageSubjects.LABEL_POOL_RELEASE,
+                                    SERIALIZER::encode, SERIALIZER::decode,
+                                    replicaInfo.master().get()));
+        }
+        return false;
+    }
+
+    private boolean internalRelease(LabelResourceRequest request) {
+        resourcePoolLock.writeLock().lock();
+        DeviceId deviceId = request.deviceId();
+        Collection<LabelResource> release = request.releaseCollection();
+        LabelResourcePool pool = resourcePool.get(deviceId).value();
+        if (pool == null) {
+            resourcePoolLock.writeLock().unlock();
+            log.info("the label resource pool of device id {} does not exist");
+            return false;
+        }
+        Set<LabelResource> storeSet = new HashSet<LabelResource>(
+                                                                 pool.releaseLabelId());
+        LabelResource labelResource = null;
+        long realReleasedNum = 0;
+        for (Iterator<LabelResource> it = release.iterator(); it.hasNext();) {
+            labelResource = it.next();
+            if (labelResource.labelResourceId().labelId() < pool.beginLabel()
+                    .labelId()
+                    || labelResource.labelResourceId().labelId() > pool
+                            .endLabel().labelId()) {
+                continue;
+            }
+            if (pool.currentUsedMaxLabelId().labelId() > labelResource
+                    .labelResourceId().labelId()
+                    || !storeSet.contains(labelResource)) {
+                storeSet.add(labelResource);
+                realReleasedNum++;
+            }
+        }
+        long beginNum = pool.beginLabel().labelId();
+        long endNum = pool.endLabel().labelId();
+        long totalNum = pool.totalNum();
+        long usedNum = pool.usedNum() - realReleasedNum;
+        long current = pool.currentUsedMaxLabelId().labelId();
+        ImmutableSet<LabelResource> s = ImmutableSet.copyOf(storeSet);
+        LabelResourcePool newPool = new LabelResourcePool(deviceId.toString(),
+                                                          beginNum, endNum,
+                                                          totalNum, usedNum,
+                                                          current, s);
+        resourcePool.put(deviceId, newPool);
+        log.info("success to release label resource");
+        resourcePoolLock.writeLock().unlock();
+        return true;
+    }
+
+    @Override
+    public boolean isDevicePoolFull(DeviceId deviceId) {
+        LabelResourcePool pool = resourcePool.get(deviceId).value();
+        if (pool == null) {
+            return true;
+        }
+        return pool.currentUsedMaxLabelId() == pool.endLabel()
+                && pool.releaseLabelId().size() == 0 ? true : false;
+    }
+
+    @Override
+    public long getFreeNumOfDevicePool(DeviceId deviceId) {
+        LabelResourcePool pool = resourcePool.get(deviceId).value();
+        if (pool == null) {
+            return 0;
+        }
+        return pool.endLabel().labelId()
+                - pool.currentUsedMaxLabelId().labelId()
+                + pool.releaseLabelId().size();
+    }
+
+    @Override
+    public LabelResourcePool getDeviceLabelResourcePool(DeviceId deviceId) {
+        return resourcePool.get(deviceId).value();
+    }
+
+    @Override
+    public boolean destroyGlobalPool() {
+        return this.internalDestroy(DeviceId
+                .deviceId(GLOBAL_RESOURCE_POOL_DEVICE_ID));
+    }
+
+    @Override
+    public Collection<LabelResource> applyFromGlobalPool(long applyNum) {
+        LabelResourceRequest request = new LabelResourceRequest(
+                                                                DeviceId.deviceId(GLOBAL_RESOURCE_POOL_DEVICE_ID),
+                                                                LabelResourceRequest.Type.APPLY,
+                                                                applyNum, null);
+        return this.internalApply(request);
+    }
+
+    @Override
+    public boolean releaseToGlobalPool(Set<LabelResourceId> release) {
+        Set<LabelResource> set = new HashSet<LabelResource>();
+        DefaultLabelResource resource = null;
+        for (LabelResourceId labelResource : release) {
+            resource = new DefaultLabelResource(
+                                                DeviceId.deviceId(GLOBAL_RESOURCE_POOL_DEVICE_ID),
+                                                labelResource);
+            set.add(resource);
+        }
+        LabelResourceRequest request = new LabelResourceRequest(
+                                                                DeviceId.deviceId(GLOBAL_RESOURCE_POOL_DEVICE_ID),
+                                                                LabelResourceRequest.Type.APPLY,
+                                                                0,
+                                                                ImmutableSet
+                                                                        .copyOf(set));
+        return this.internalRelease(request);
+    }
+
+    @Override
+    public boolean isGlobalPoolFull() {
+        return this.isDevicePoolFull(DeviceId
+                .deviceId(GLOBAL_RESOURCE_POOL_DEVICE_ID));
+    }
+
+    @Override
+    public long getFreeNumOfGlobalPool() {
+        return this.getFreeNumOfDevicePool(DeviceId
+                .deviceId(GLOBAL_RESOURCE_POOL_DEVICE_ID));
+    }
+
+    @Override
+    public LabelResourcePool getGlobalLabelResourcePool() {
+        return this.getDeviceLabelResourcePool(DeviceId
+                .deviceId(GLOBAL_RESOURCE_POOL_DEVICE_ID));
+    }
+
+    private <T> T complete(Future<T> future) {
+        try {
+            return future.get(PEER_REQUEST_TIMEOUT_MS,
+                                                TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            log.error("Interrupted while waiting for operation to complete.", e);
+            return null;
+        } catch (TimeoutException | ExecutionException e) {
+            log.error("Failed remote operation", e);
+            return null;
+        }
+    }
+}

+package org.onosproject.store.resource.impl;
+
+import org.onosproject.store.cluster.messaging.MessageSubject;
+
+public final class LabelResourceMessageSubjects {
+
+    private LabelResourceMessageSubjects() {
+    }
+    public static final MessageSubject LABEL_POOL_CREATED
+                        = new MessageSubject("label-resource-pool-created");
+    public static final MessageSubject LABEL_POOL_DESTROYED
+                        = new MessageSubject("label-resource-pool-destroyed");
+    public static final MessageSubject LABEL_POOL_APPLY
+                        = new MessageSubject("label-resource-pool-apply");
+    public static final MessageSubject LABEL_POOL_RELEASE
+                        = new MessageSubject("label-resource-pool-release");
+}