Add HA.py

A set of mininet based HA tests based on onos.py
Currently includes the following tests:
    - a control network partitioning test
    - A dynamic cluster scaling test

Change-Id: I9a8e1019dcc51666fee1d933afd66ff390592525

+#!/usr/bin/env python
+
+"""
+NOTES
+
+To change onos log levels before start you can add something similar to
+onos.py's ONOSNode.start method before starting onos service:
+  # Change log levels
+  self.ucmd( 'echo "log4j.logger.io.atomix= DEBUG" >> $ONOS_HOME/apache-karaf-*/etc/org.ops4j.pax.logging.cfg' )
+
+"""
+
+import argparse
+from mininet.log import output, info, warn, error, debug, setLogLevel
+from mininet.cli import CLI as origCLI
+from mininet.net import Mininet
+from mininet.topo import SingleSwitchTopo, Topo
+from mininet.node import Host
+from os.path import join
+from glob import glob
+import re
+import json
+from collections import deque
+import hashlib
+import onos  # onos.py
+
+# Utility functions
+
+def pause( net, msg, hint=False):
+    """Reenter the CLI. Note that we use the mn base CLI class to allow
+    extensibility and combination of custom files"""
+
+    info( msg )
+
+    if hint:
+        help_msg = "Currently in the root Mininet net namespace...\n"
+        help_msg += "To access control net functions use:\n"
+        help_msg += "\tpx cnet=net.controllers[0].net\n"
+        help_msg += "\tpy cnet.METHOD\n"
+        help_msg += "To send commands to each onos node, use: onos_all CMD\n"
+        help_msg += "\nBy default, ONOS nodes are running on the 192.168.123.X network\n"
+        info( "%s\n" % help_msg )
+    # NOTE: If we use onos.py as a custom file and as an imported module,
+    #       we get two different sets of ONOS* classes. They don't play
+    #       together and things don't work properly. Specifically the
+    #       isinstance calls fail. This is due to import vs. exec calls
+    onos.ONOSCLI( net )
+
+def cprint( msg, color="default"):
+    color=color.lower()
+    colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
+               'blue': '\033[94m', 'green': '\033[92m',
+               'yellow': '\033[93m', 'red': '\033[91m',
+               'end': '\033[0m' }
+    pre = colors.get( color, '' )
+    output = pre + msg + colors['end']
+    print( output )
+
+def getNode( net, nodeId=0 ):
+    "Helper function: return ONOS node, defaults to the first node"
+    return net.controllers[ 0 ].nodes()[ nodeId ]
+
+def onos_cli( net, line, nodeId=0 ):
+    "Send command to ONOS CLI"
+    c0 = net.controllers[ 0 ]
+    # FIXME add this back after import onos.py works
+    if isinstance( c0, onos.ONOSCluster ):
+        # cmdLoop strips off command name 'onos'
+        if line.startswith( ':' ):
+            line = 'onos' + line
+        node = getNode( net, nodeId )
+        if line:
+            line = '"%s"' % line
+        cmd = 'client -h %s %s' % ( node.IP(), line )
+        #node.cmdPrint( cmd )
+        output  = node.cmd( cmd )
+        info( line )
+        # Remove verbose spam from output
+        m = re.search( "unverified \{\} key: \{\}", output )
+        if m:
+            info( output[m.end():] )
+        else:
+            info( output )
+
+def onos_all( net, line ):
+    onosNodes = [ n for cluster in net.controllers for n in cluster.nodes() ]
+    for node in range( len( onosNodes ) ):
+        cprint( "*" * 53, "red" )
+        cprint( "onos%s: %s" % ( str( node + 1 ), repr( onosNodes[ node ] ) ),
+                "red" )
+        cprint( "*" * 53, "red" )
+        onos_cli( net, line, node )
+
+# FIXME This needs a better name
+def do_onos_all( self, line ):
+    onos_all( self.mn, line)
+
+# Add custom cli commands
+# NOTE: This is so we can keep ONOSCLI and also add commands to it!
+origCLI.do_onos_all = do_onos_all
+
+# Test cases
+
+def Partition( net ):
+    # Controller net instance
+    cnet = net.controllers[0].net
+
+    info( "ONOS control network partition test\n")
+    net.pingAll()
+    if args.interactive:
+        pause( net,  "~~~ Dropping into cli... Exit cli to continue test\n", True )
+
+    onos_all( net, "nodes;partitions;partitions -c")
+    info( "~~~ Right before the partitioned\n" )
+    if args.interactive:
+        pause( net, "Dropping into cli... Exit cli to continue test\n" )
+
+    cs1 = cnet.switches[0]
+    cs2 = cnet.switches[1]
+
+    # PARTITION sub-clusters
+
+    # we need to use names here
+    cnet.configLinkStatus( cs1.name, cs2.name, "down" )
+    onos_all( net, "nodes;partitions;partitions -c")
+    info( "~~~ Right after cluster is partitioned. Next step is to heal the partition\n" )
+    if args.interactive:
+        pause( net, "Dropping into cli... Exit cli to continue test\n" )
+
+    cnet.configLinkStatus( cs1.name, cs2.name, "up" )
+    onos_all( net, "nodes;partitions;partitions -c")
+    info( "~~~ Right after the partition is healed \n" )
+    if args.interactive:
+        pause( net, "Test is finished! Exit cli to exit test.\n" )
+
+def Scaling( net ):
+
+    def startNodes( net, nodes ):
+        "start multiple ONOS nodes"
+        cluster = net.controllers[0]
+        cluster.activeNodes.extend( nodes )
+        cluster.activeNodes = sorted( set( cluster.activeNodes ) )
+        for node in nodes:
+            node.shouldStart = True
+            node.start( cluster.env, cluster.activeNodes )
+        for node in nodes:
+            node.waitStarted()
+
+    # control net objects
+    cluster = net.controllers[0]
+    cnet = cluster.net
+    cs1 = cnet.switches[0]
+
+    info( "ONOS dynamic clustering scaling test\n")
+    # Start the first node
+    cluster.activeNodes.append( cnet.hosts[0] )
+    cluster.activeNodes = sorted( set( cluster.activeNodes ) )
+    startNodes( net, cluster.activeNodes )
+
+    onos_all( net, "nodes;partitions;partitions -c")
+    if args.interactive:
+        pause( net, "Dropping into cli... Exit cli to continue test\n" )
+
+    # Scale up by two
+    while True:
+        new = [ n for c in net.controllers for n in c.net.hosts if isinstance( n, DynamicONOSNode) and not n.started ][:2]
+        if not new:
+            break
+        startNodes( net, new )
+        onos_all( net, "nodes;partitions;partitions -c")
+        if args.interactive:
+            pause( net, "Dropping into cli... Exit cli to continue test\n" )
+
+    # Scale down
+    for i in range( len( cluster.activeNodes ) - 1 ):
+        node = cluster.activeNodes.pop()
+        node.genPartitions( cluster.activeNodes, node.metadata )
+        onos_all( net, "nodes;partitions;partitions -c")
+        if args.interactive:
+            pause( net, "Dropping into cli... Exit cli to continue test\n" )
+    if args.interactive:
+        pause( net, "Test is finished! Exit cli to exit test.\n" )
+
+
+
+# Mininet object subclasses
+
+class HTTP( Host ):
+    def __init__( self, *args, **kwargs ):
+        super( HTTP, self).__init__( *args, **kwargs )
+        self.dir = '/tmp/%s' % self.name
+        self.cmd( 'rm -rf', self.dir )
+        self.cmd( 'mkdir', self.dir )
+        self.cmd( 'cd', self.dir )
+
+    def start( self ):
+        output( "(starting HTTP Server)" )
+        # start python web server as a bg process
+        self.cmd( 'python -m SimpleHTTPServer &> web.log &' )
+
+    def stop( self ):
+        # XXX is this ever called?
+        print "Stopping HTTP Server..."
+        print self.cmd( 'fg' )
+        print self.cmd( '\x03' )  # ctrl-c
+
+
+class DynamicONOSNode( onos.ONOSNode ):
+    def __init__( self, *args, **kwargs ):
+        self.shouldStart = False
+        self.started = False
+        self.metadata = '/tmp/cluster.json'
+        super( DynamicONOSNode, self ).__init__( *args, **kwargs )
+        # XXX HACK, need to get this passed in correctly
+        self.alertAction = 'warn'
+
+    def start( self, env, nodes=()):
+        if not self.shouldStart:
+            return
+        elif self.started:
+            return
+        else:
+            ##### Modified from base class
+            env = dict( env )
+            env.update( ONOS_HOME=self.ONOS_HOME )
+            if self.remote:
+                # Point onos to rewmote cluster metadata file
+                ip = self.remote.get( 'ip', '127.0.0.1' )
+                port = self.remote.get( 'port', '8000' )
+                filename = self.remote.get( 'filename', 'cluster.json' )
+                remote = 'http://%s:%s/%s' % ( ip, port, filename )
+                uri = '-Donos.cluster.metadata.uri=%s' % remote
+                prev = env.get( 'JAVA_OPTS', False )
+                if prev:
+                    jarg = ':'.join( [prev, uri] )
+                else:
+                    jarg = uri
+                env.update( JAVA_OPTS=jarg )
+            self.updateEnv( env )
+            karafbin = glob( '%s/apache*/bin' % self.ONOS_HOME )[ 0 ]
+            onosbin = join( self.ONOS_ROOT, 'tools/test/bin' )
+            self.cmd( 'export PATH=%s:%s:$PATH' % ( onosbin, karafbin ) )
+            self.cmd( 'cd', self.ONOS_HOME )
+            self.ucmd( 'mkdir -p config ' )
+            self.genPartitions( nodes, self.metadata )
+            info( '(starting %s)' % self )
+            service = join( self.ONOS_HOME, 'bin/onos-service' )
+            self.ucmd( service, 'server 1>../onos.log 2>../onos.log'
+                       ' & echo $! > onos.pid; ln -s `pwd`/onos.pid ..' )
+            self.onosPid = int( self.cmd( 'cat onos.pid' ).strip() )
+            self.warningCount = 0
+            ####
+            self.started=True
+
+    def sanityCheck( self, lowMem=100000 ):
+        if self.started:
+            super( DynamicONOSNode, self ).sanityCheck( lowMem )
+
+    def waitStarted( self ):
+        if self.started:
+            super( DynamicONOSNode, self ).waitStarted()
+
+    def genPartitions( self, nodes, location='/tmp/cluster.json' ):
+        """
+        Generate a cluster metadata file for dynamic clustering.
+        Note: name should be the same in different versions of the file as
+              well as the number of partitions.
+        """
+        def genParts( nodes, k, parts=3):
+            l = deque( nodes )
+            perms = []
+            for i in range( 1, parts + 1 ):
+                part = {
+                           'id': i,
+                           'members': list(l)[:k]
+                       }
+                perms.append( part )
+                l.rotate( -1 )
+            return perms
+
+        print "Generating %s with %s" % ( location, str(nodes) )
+        port = 9876
+        ips = [ node.IP() for node in nodes ]
+        node = lambda k: { 'id': k, 'ip': k, 'port': port }
+        m = hashlib.sha256( "Mininet based ONOS test" )
+        name = int(m.hexdigest()[:8], base=16 )
+        partitions = genParts( ips, 3 )
+        data = {
+                'name': name,
+                'nodes': [ node(v) for v in ips ],
+                'partitions': partitions
+               }
+        output = json.dumps( data, indent=4 )
+        with open( location, 'w' ) as f:
+            f.write( output )
+        cprint( output, "yellow" )
+
+
+class DynamicONOSCluster( onos.ONOSCluster ):
+    def __init__( self, *args, **kwargs ):
+        self.activeNodes = []
+        # TODO: can we get super to use super's nodes()?
+        super( DynamicONOSCluster, self ).__init__( *args, **kwargs )
+        self.activeNodes = [ h for h in self.net.hosts if onos.isONOSNode( h ) ]
+        onos.updateNodeIPs( self.env, self.nodes() )
+        self.activeNodes = []
+
+    def start( self ):
+        "Start up ONOS control network"
+        info( '*** ONOS_APPS = %s\n' % onos.ONOS_APPS )
+        self.net.start()
+        for node in self.net.hosts:
+            if onos.isONOSNode( node ):
+                node.start( self.env, self.nodes() )
+            else:
+                try:
+                    node.start()
+                except AttributeError:
+                    # NAT doesn't have start?
+                    pass
+        info( '\n' )
+        self.configPortForwarding( ports=self.forward, action='A' )
+        self.waitStarted()
+        return
+
+    def nodes( self ):
+        "Return list of ONOS nodes that should be running"
+        return self.activeNodes
+
+class HATopo( Topo ):
+    def build( self, partitions=[], serverCount=1, dynamic=False, **kwargs ):
+        """
+        partitions  = a list of strings specifing the assignment of onos nodes
+                      to regions. ['1', '2,3'] designates two regions, with
+                      ONOS 1 in the first and ONOS 2 and 3 in the second.
+        serverCount = If partitions is not given, then the number of ONOS
+                      nodes to create
+        dynamic     = A boolean indicating dynamic ONOS clustering
+        """
+        self.switchNum = 1
+        if dynamic:
+            cls = DynamicONOSNode
+        else:
+            cls = onos.ONOSNode
+        if partitions:
+            prev = None
+            for partition in partitions:
+                # Create a region of ONOS nodes connected to a switch
+                # FIXME Check for nodes that are not assigned to a partition?
+                cur = self.addRegion( partition, cls )
+
+                # Connect switch to previous switch
+                if prev:
+                    self.addLink( prev, cur )
+                prev = cur
+        else:
+            partition = ','.join( [ str( x ) for x in range( 1, serverCount + 1 ) ] )
+            cs1 = self.addRegion( partition, cls )
+        if dynamic:
+            # TODO Pass these in
+            scale = 2
+            new = ','.join( [ str( x + 1 ) for x in range( serverCount , serverCount + scale ) ] )
+            cs2 = self.addRegion( new, cls )
+            self.addLink( cs1, cs2 )
+            server = self.addHost( "server", cls=HTTP )
+            for switch in self.switches():
+                self.addLink( server, switch )
+
+    def addRegion( self, partition, cls=onos.ONOSNode ):
+        switch = self.addSwitch( 'cs%s' % self.switchNum )
+        self.switchNum += 1
+        for n in partition.split( ',' ):
+            node = self.addHost( "onos" + str( n ), cls=cls )
+            self.addLink( switch, node )
+        return switch
+
+
+CLI = onos.ONOSCLI
+
+# The main runner
+def runTest( args ):
+    test = None
+    if args.test == "partition":
+        test=Partition
+        serverCount = args.nodes
+        # NOTE we are ignoring serverCount for this test, using partition assignment instead.
+        topo = HATopo( partitions=args.partition )
+        # FIXME Configurable dataplane topology
+        net = Mininet( topo=SingleSwitchTopo( 3 ),
+                       controller=[ onos.ONOSCluster( 'c0', topo=topo, alertAction='warn' ) ],
+                       switch=onos.ONOSOVSSwitch )
+    elif args.test == "scaling":
+        test=Scaling
+        serverCount = args.nodes
+        topo = HATopo( serverCount=serverCount, dynamic=True )
+        net = Mininet( topo=SingleSwitchTopo( 3 ),
+                       controller=[ DynamicONOSCluster( 'c0', topo=topo, alertAction='warn' ) ],
+                       switch=onos.ONOSOVSSwitch )
+        cluster = net.controllers[0]
+        cnet = cluster.net
+        server = cnet.get( 'server' )
+        remote = { 'ip': server.IP(),
+                   'port': '8000',
+                   'filename':'cluster.json' }
+        for node in cnet.hosts:
+            if isinstance( node, DynamicONOSNode ):
+                node.metadata = '%s/cluster.json' % server.dir
+                node.remote = remote
+        ips = []
+        cluster.activeNodes = [ cnet.get( "onos%s" % ( i + 1 ) ) for i in range( serverCount ) ]
+        for node in cluster.activeNodes:
+            node.shouldStart = True
+    else:
+        print "Incorrect test"
+        return
+    net.start()
+    if args.interactive:
+        CLI( net )
+    test(net)
+    CLI( net )
+    net.stop()
+
+
+if __name__ == '__main__':
+    setLogLevel( 'info' )
+    # Base parser
+    parser= argparse.ArgumentParser(
+            description='Mininet based HA tests for ONOS. For more detailed help on a test include the test option' )
+    parser.add_argument(
+            '-n', '--nodes', metavar="NODES", type=int, default=1,
+            help="Number of nodes in the ONOS cluster" )
+    parser.add_argument(
+            '-i', '--interactive',# type=bool,
+            default=False, action="store_true",
+            help="Pause the test in between steps" )
+    test_parsers=parser.add_subparsers( title="Tests", help="Types of HA tests", dest="test" )
+
+    # Partition test parser
+    partition_help = 'Network partition test. Each set of ONOS nodes is connected to their own switch in the control network. Partitions are introduced by removing links between control network switches.'
+    partition_parser = test_parsers.add_parser(
+            "partition", description=partition_help )
+    partition_parser.add_argument(
+            '-p', '--partition', metavar='Partition', required=True,
+            type=str, nargs=2,
+            help='Specify the membership for two partitions by node id. Nodes are comma separated and node count begins at 1. E.g. "1,3 2" will create a network with 3 ONOS nodes and two connected switches. Switch 1 will be connected to ONOS1 and ONOS3 while switch 2 will be connected to ONOS2. A partition will be created by disconnecting the two switches. All ONOS nodes will still be connected to the dataplane.' )
+
+    # Dynamic scaling test parser
+    # FIXME Replace with real values
+    scaling_parser = test_parsers.add_parser( "scaling" )
+
+    args = parser.parse_args()
+    runTest( args )

@@ -117,7 +117,8 @@ def updateNodeIPs( env, nodes ):
     for index, node in enumerate( nodes, 1 ):
     for index, node in enumerate( nodes, 1 ):
         var = 'OC%d' % index
         var = 'OC%d' % index
         env[ var ] = node.IP()
         env[ var ] = node.IP()
-    env[ 'OCI' ] = env[ 'OCN' ] = env[ 'OC1' ]
+    if nodes:
+        env[ 'OCI' ] = env[ 'OCN' ] = env[ 'OC1' ]
     env[ 'ONOS_INSTANCES' ] = '\n'.join(
     env[ 'ONOS_INSTANCES' ] = '\n'.join(
         node.IP() for node in nodes )
         node.IP() for node in nodes )
     environ.update( env )
     environ.update( env )
@@ -242,6 +243,7 @@ class ONOSNode( Controller ):
         self.ONOS_HOME = '/tmp'
         self.ONOS_HOME = '/tmp'
         self.cmd( 'rm -rf', self.dir )
         self.cmd( 'rm -rf', self.dir )
         self.ONOS_HOME = unpackONOS( self.dir, run=self.ucmd )
         self.ONOS_HOME = unpackONOS( self.dir, run=self.ucmd )
+        self.ONOS_ROOT = ONOS_ROOT
 
 
     # pylint: disable=arguments-differ
     # pylint: disable=arguments-differ