ONOS-3422 inter-domain cross connect

- Add network configuration about cross connect port for CO-ONOS
- revised metro.py requires ecord.co app after
  (Change-Id: I3892e780bc6550f8a8d8be622b9fee5322c1dab5)
  to be loaded.
- stop using onos-topo-cfg to send netcfg

Change-Id: Ie90e69c4134d1f71893bf43ee6c290bdbd273aeb

@@ -83,6 +83,7 @@ public class BasicNetworkConfigs {
                     return new BasicLinkConfig();
                 }
             },
+            // TODO move this optical specific configuration out to optical app
             new ConfigFactory<ConnectPoint, OpticalPortConfig>(CONNECT_POINT_SUBJECT_FACTORY,
                                                                OpticalPortConfig.class,
                                                                "optical") {

 #!/usr/bin/env python
 
+import json
+
 from mininet.net import Mininet
 from mininet.node import UserSwitch, DefaultController, RemoteController, Host
 from mininet.topo import Topo
-from mininet.log import setLogLevel, info
+from mininet.log import  setLogLevel, info, error, warn
 from mininet.cli import CLI
 from mininet.link import OVSIntf
+from mininet.util import quietRun
 
 from opticalUtils import LINCSwitch, LINCLink
 
@@ -33,6 +36,7 @@ class Domain(object):
         self.__ctrls[name] = args if args else {}
         return name
 
+    # Note: This method will return the name of the swich, not the switch object
     def addSwitch(self, name, **args):
         self.__switches[name] = args if args else {}
         return name
@@ -90,10 +94,12 @@ class OpticalDomain(Domain):
             oean = { "optical.regens": 0 }
             self.addSwitch('OE%s' % i, dpid='0000ffffffffff0%s' % i, annotations=oean, cls=LINCSwitch)
 
+        # ROADM port number OE"1" -> OE'2' = "1"'2'00
+        # leaving port number up to 100 open for use by Och port
         an = { "durable": "true" }
-        self.addLink('OE1', 'OE2', port1=50, port2=30, annotations=an, cls=LINCLink)
-        self.addLink('OE2', 'OE3', port1=50, port2=30, annotations=an, cls=LINCLink)
-        self.addLink('OE3', 'OE1', port1=50, port2=30, annotations=an, cls=LINCLink)
+        self.addLink('OE1', 'OE2', port1=1200, port2=2100, annotations=an, cls=LINCLink)
+        self.addLink('OE2', 'OE3', port1=2300, port2=3200, annotations=an, cls=LINCLink)
+        self.addLink('OE3', 'OE1', port1=3100, port2=1300, annotations=an, cls=LINCLink)
 
 class FabricDomain(Domain):
     """
@@ -139,10 +145,11 @@ class FabricDomain(Domain):
         domains to the core.  name: the UserSwitch to connect the OVS to.
         """
         self.__tether = self.addSwitch(tname, dpid=tdpid)
+        # Note: OVS port number '1' reserved for port facing the fabric
         self.addLink(tname, name, port1=1)
 
     def getTether(self):
-        """ get connection point of this fabric to the core """
+        """ get the switch name of this fabric facing the core """
         return self.__tether
 
 
@@ -157,10 +164,6 @@ class IpHost(Host):
         self.cmd(mtu)
         self.cmd('ip route add default via %s' % self.gateway)
 
-# fixed port numbers for attachment points (APs) between CORD and metro domains
-OVS_AP=2
-OE_AP=10
-
 def setup(argv):
     domains = []
     ctlsets = sys.argv[1:]
@@ -180,6 +183,16 @@ def setup(argv):
         for j in range (len(ctls)):
             f.addController('c%s%s' % (i,j), controller=RemoteController, ip=ctls[j])
 
+    # netcfg for each domains
+    # Note: Separate netcfg for domain0 is created in opticalUtils
+    domainCfgs = []
+    for i in range (0,len(ctlsets)):
+        cfg = {}
+        cfg['devices'] = {}
+        cfg['ports'] = {}
+        cfg['links'] = {}
+        domainCfgs.append(cfg)
+
     # make/setup Mininet object
     net = Mininet()
     for d in domains:
@@ -187,10 +200,19 @@ def setup(argv):
         d.injectInto(net)
 
     # connect COs to core - sort of hard-wired at this moment
+    # adding cross-connect links
     for i in range(1,len(domains)):
-        an = { "bandwidth": 100000, "durable": "true" }
-        net.addLink(domains[i].getTether(), d0.getSwitches('OE%s' % i),
-                    port1=OVS_AP, port2=OE_AP, speed=10000, annotations=an, cls=LINCLink)
+        # add 10 cross-connect links between domains
+        xcPortNo=2
+        ochPortNo=10
+        for j in range(0, 10):
+            an = { "bandwidth": 10, "durable": "true" }
+            net.addLink(domains[i].getTether(), d0.getSwitches('OE%s' % i),
+                        port1=xcPortNo+j, port2=ochPortNo+j, speed=10000, annotations=an, cls=LINCLink)
+
+            xcId = 'of:' + domains[i].getSwitches(name=domains[i].getTether()).dpid + '/' + str(xcPortNo+j)
+            ochId = 'of:' + d0.getSwitches('OE%s' % i).dpid + '/' + str(ochPortNo+j)
+            domainCfgs[i]['ports'][xcId] = {'cross-connect': {'remote': ochId}}
 
     # fire everything up
     net.build()
@@ -203,6 +225,22 @@ def setup(argv):
     cfgnet.controllers = d0.getControllers()
     LINCSwitch.bootOE(cfgnet, d0.getSwitches())
 
+    # send netcfg json to each CO-ONOS
+    for i in range(1,len(domains)):
+        info('*** Pushing Topology.json to CO-ONOS %d\n' % i)
+        filename = 'Topology%d.json' % i
+        with open(filename, 'w') as outfile:
+            json.dump(domainCfgs[i], outfile, indent=4, separators=(',', ': '))
+
+        output = quietRun('%s/tools/test/bin/onos-netcfg %s %s &'\
+                           % (LINCSwitch.onosDir,
+                              domains[i].getControllers()[0].ip,
+                              filename), shell=True)
+        # successful output contains the two characters '{}'
+        # if there is more output than this, there is an issue
+        if output.strip('{}'):
+            warn('***WARNING: Could not push topology file to ONOS: %s\n' % output)
+
     CLI(net)
     net.stop()
     LINCSwitch.shutdownOE()

@@ -3,7 +3,7 @@
 '''
 Notes:
 
-This file contains classes and methods useful for integrating LincOE with Mininet, 
+This file contains classes and methods useful for integrating LincOE with Mininet,
 such as startOE, stopOE, LINCLink, and OpticalSwitch
 
 - $ONOS_ROOT ust be set
@@ -24,7 +24,7 @@ such as startOE, stopOE, LINCLink, and OpticalSwitch
         ------------
     - import LINCLink and OpticalSwitch from this module
     - import startOE and stopOE from this module
-    - create topology as you would a normal topology. when 
+    - create topology as you would a normal topology. when
       to an optical switch with topo.addLink, always specify cls=LINCLink
     - when creating an optical switch, use cls=OpticalSwitch in topo.addSwitch
     - for annotations on links and switches, a dictionary must be passed in as
@@ -41,13 +41,13 @@ we need to add another object to Mininet that contains all of the json object
 information for each switch. We would still subclass switch and link, but these
 classes would basically be dummy classes that store their own json information
 in the Mininet class object. We may also change the default switch class to add
-it's tap interfaces from lincOE during startup. The start() method for mininet would 
+it's tap interfaces from lincOE during startup. The start() method for mininet would
 grab all of the information from these switches and links, write configuration files
 for lincOE using the json module, start lincOE, then run the start methodfor each
 switch. The new start() method for each switch would parse through the sys.config
-file that was created and find the tap interface it needs to connect to, similar 
-to the findTap function that I currently use. After all of the controllers and 
-switches have been started, the new Mininet start() method should also push the 
+file that was created and find the tap interface it needs to connect to, similar
+to the findTap function that I currently use. After all of the controllers and
+switches have been started, the new Mininet start() method should also push the
 Topology configuration file to ONOS.
 
 '''
@@ -119,7 +119,7 @@ class LINCSwitch(OpticalSwitch):
                     if dpid:
                         dpids_to_ids[dpid.group().replace(':', '')] = switch_id
                         switch_id += 1
-            return dpids_to_ids     
+            return dpids_to_ids
         except:
             print "Error working with {}\nError: {}\n".format(sysConfig, sys.exc_info())
             fd.close()
@@ -287,8 +287,8 @@ class LINCSwitch(OpticalSwitch):
         with open("crossConnect.json", 'w') as fd:
             json.dump(crossConnectJSON, fd, indent=4, separators=(',', ': '))
         info('*** Pushing crossConnect.json to ONOS\n')
-        output = quietRun('%s/tools/test/bin/onos-topo-cfg %s\
-         Topology.json network/configuration/' % (self.onosDir, self.controllers[ 0 ].ip), shell=True)
+        output = quietRun('%s/tools/test/bin/onos-netcfg %s\
+         Topology.json' % (self.onosDir, self.controllers[ 0 ].ip), shell=True)
 
     def stop_oe(self):
         '''
@@ -388,7 +388,7 @@ class LINCSwitch(OpticalSwitch):
             json.dump(topoJSON, outfile, indent=4, separators=(',', ': '))
 
         info('*** Converting Topology.json to linc-oe format (TopoConfig.json) file (no oecfg) \n')
-        
+
         topoConfigJson = {}
 
         topoConfigJson["switchConfig"] = LINCSwitch.getSwitchConfig(net.switches)
@@ -455,21 +455,20 @@ class LINCSwitch(OpticalSwitch):
         opener.open(url)
         urllib2.install_opener(opener)
         # focus on just checking the state of devices we're interested in
-        devlist =  map( lambda x: x['uri'], devices )
+        # expected devices availability map
+        devMap =  dict.fromkeys(map( lambda x: x['uri'], devices ), False)
         while True:
             response = json.load(urllib2.urlopen(url))
             devs = response.get('devices')
 
-            # Wait for all devices to be registered. There is a chance that this is only a subgraph.
-            if (len(devices) == len(devs)):
+            # update availability map
+            for d in devs:
+                if devMap.has_key(d['id']):
+                    devMap[d['id']] = d['available']
 
-                # Wait for all devices to available
-                available = True
-                for d in devs:
-                    if d['id'] in devlist:
-                        available &= d['available']
-                if available:
-                    break
+            # Check if all devices we're interested became available
+            if all(devMap.viewvalues()):
+                break;
 
             if (time >= TIMEOUT):
                 error('***ERROR: ONOS did not register devices within %s seconds\n' % TIMEOUT)
@@ -480,7 +479,7 @@ class LINCSwitch(OpticalSwitch):
 
         info('*** Pushing Topology.json to ONOS\n')
         for index in range(len(LINCSwitch.controllers)):
-            output = quietRun('%s/tools/test/bin/onos-topo-cfg %s Topology.json network/configuration/ &'\
+            output = quietRun('%s/tools/test/bin/onos-netcfg %s Topology.json &'\
                                % (LINCSwitch.onosDir, LINCSwitch.controllers[ index ].ip), shell=True)
             # successful output contains the two characters '{}'
             # if there is more output than this, there is an issue
@@ -539,7 +538,7 @@ class LINCSwitch(OpticalSwitch):
     @staticmethod
     def getSwitchConfig(switches):
         switchConfig = []
-        
+
         # Iterate through all switches and convert the ROADM switches to linc-oe format
         for switch in switches:
             if isinstance(switch, LINCSwitch):
@@ -566,7 +565,7 @@ class LINCSwitch(OpticalSwitch):
     @staticmethod
     def getLinkConfig(links):
         linkConfig = []
-        
+
         # Iterate through all non-edge links and convert them to linc-oe format
         for link in links:
             if isinstance(link, LINCLink):
@@ -581,7 +580,7 @@ class LINCSwitch(OpticalSwitch):
                 params = {}
                 params["nodeName1"] = link.intf1.node.name
                 params["nodeName2"] = link.intf2.node.name
-                
+
                 params["port1"] = link.port1
                 params["port2"]  = link.port2
 
@@ -609,7 +608,7 @@ class LINCSwitch(OpticalSwitch):
         for link in net.links:
             if isinstance(link, LINCLink) and link.isCrossConnect():
                 tapCount += 1
-                    
+
         while True:
             # tapCount can be less than the actual number of taps if the optical network
             # is a subgraph of a larger multidomain network.
@@ -722,7 +721,7 @@ class LINCLink(Link):
                 node1.crossConnects.append(self)
         else:
             cls1 = Intf
-            # bad hack to stop error message from appearing when we try to set up intf in a packet switch, 
+            # bad hack to stop error message from appearing when we try to set up intf in a packet switch,
             # and there is no interface there( because we do not run makeIntfPair ). This way, we just set lo up
             intfName1 = 'lo'
         if isinstance(node2, LINCSwitch):