Added onos-cluster-execute-expect as a cluster version of onos-execute-expect + …

…updated stc scenarios for dist primitives to use this for verification

Change-Id: Ife3bde21118ebd21e8abf807070a72ac58341120

+#!/bin/bash
+# -----------------------------------------------------------------------------
+# Executes a command on all ONOS instances in a cluster and matches the output
+# from each instance to the passed one.
+# First argument is the command and it's arguments if needed, then --expect and
+# after it the string of what the output should be.
+# Example:
+# onos-cluster-execute-expect fooCommand fooParamenter --expect fooOutputString
+# -----------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+
+aux=/tmp/stc-$$.log
+trap "rm -f $aux 2>/dev/null" EXIT
+cmd=""
+for a in ${*:1}; do shift; if [ "$a" = "--expect" ]; then break; fi; cmd="$cmd $a"; done
+expect="${@: -1}"
+echo $cmd
+echo $expect
+node_count=`onos $OC1 nodes | wc -l`
+for i in `seq 1 $node_count`; do
+    node_var="OC$i"
+    onos ${!node_var} $cmd > $aux
+    cat $aux
+    grep -q $expect $aux || exit 1
+done
+echo "expected value found"
+exit 0

@@ -25,13 +25,13 @@
             exec="onos ${OCI} app activate org.onosproject.distributedprimitives"/>
 
         <step name="Test-Counter-Initial-Value" requires="^"
-              exec="onos-execute-expect ${OCI} counter-test test-counter get --expect 0"/>
+              exec="onos-cluster-execute-expect counter-test test-counter get --expect 0"/>
 
         <step name="Test-Counter-Set" requires="^"
               exec="onos ${OCI} counter-test test-counter set 1"/>
 
         <step name="Test-Counter-Get" requires="^"
-              exec="onos-execute-expect ${OCI} counter-test test-counter get --expect 1"/>
+              exec="onos-cluster-execute-expect counter-test test-counter get --expect 1"/>
 
         <step name="Test-Counter-IncrementAndGet" requires="^"
               exec="onos-execute-expect ${OCI} counter-test test-counter incrementAndGet --expect 2"/>
@@ -40,7 +40,7 @@
               exec="onos-execute-expect ${OCI} counter-test test-counter getAndIncrement --expect 2"/>
 
         <step name="Test-Counter-Incremented" requires="^"
-              exec="onos-execute-expect ${OCI} counter-test test-counter get --expect 3"/>
+              exec="onos-cluster-execute-expect counter-test test-counter get --expect 3"/>
 
         <step name="Test-Counter-AddAndGet" requires="^"
               exec="onos-execute-expect ${OCI} counter-test test-counter addAndGet 10 --expect 13"/>
@@ -49,19 +49,25 @@
               exec="onos-execute-expect ${OCI} counter-test test-counter getAndAdd 10 --expect 13"/>
 
         <step name="Test-Counter-Updated-After-GetAndAdd" requires="^"
-              exec="onos-execute-expect ${OCI} counter-test test-counter get --expect 23"/>
+              exec="onos-cluster-execute-expect counter-test test-counter get --expect 23"/>
 
         <step name="Test-Counter-CompareAndSet-False" requires="^"
               exec="onos-execute-expect ${OCI} counter-test test-counter compareAndSet 1 2 --expect false"/>
 
         <step name="Test-Counter-Not-Updated-After-CAS" requires="^"
-              exec="onos-execute-expect ${OCI} counter-test test-counter get --expect 23"/>
+              exec="onos-cluster-execute-expect counter-test test-counter get --expect 23"/>
 
         <step name="Test-Counter-CompareAndSet-True" requires="^"
               exec="onos-execute-expect ${OCI} counter-test test-counter compareAndSet 23 25 --expect true"/>
 
         <step name="Test-Counter-Updated-After-CAS" requires="^"
-              exec="onos-execute-expect ${OCI} counter-test test-counter get --expect 25"/>
+              exec="onos-cluster-execute-expect counter-test test-counter get --expect 25"/>
+
+        <step name="Test-Counter-Reset" requires="^"
+              exec="onos ${OCI} counter-test test-counter set 0"/>
+
+        <step name="Test-Counter-Get-After-Reset" requires="^"
+              exec="onos-cluster-execute-expect counter-test test-counter get --expect 0"/>
 
         <!--Check with check logs-->
         <step name="Check-Log-Exceptions" requires="^"

@@ -25,7 +25,7 @@
             exec="onos ${OCI} app activate org.onosproject.distributedprimitives"/>
 
         <step name="Test-Initial-No-Leader" requires="^"
-              exec="onos-execute-expect ${OCI} leader-test test-elector show foo --expect leader=none#candidates=none"/>
+              exec="onos-cluster-execute-expect leader-test test-elector show foo --expect leader=none#candidates=none"/>
 
         <step name="Test-Leader-Run" requires="^"
               exec="onos-execute-expect ${OCI} leader-test test-elector run foo --expect leader=${OCI}#term=1#candidates=${OCI}"/>
@@ -34,7 +34,7 @@
               exec="onos ${OCI} leader-test test-elector withdraw foo"/>
 
         <step name="Test-No-Leader-After-Withdraw" requires="^"
-              exec="onos-execute-expect ${OCI} leader-test test-elector show foo --expect leader=none#candidates=none"/>
+              exec="onos-cluster-execute-expect leader-test test-elector show foo --expect leader=none#candidates=none"/>
 
         <!--Check with check logs-->
         <step name="Check-Log-Exceptions" requires="^"

@@ -25,13 +25,13 @@
             exec="onos ${OCI} app activate org.onosproject.distributedprimitives"/>
 
         <step name="Test-Map-Get" requires="^"
-              exec="onos-execute-expect ${OCI} map-test foo get a --expect null"/>
+              exec="onos-cluster-execute-expect map-test foo get a --expect null"/>
 
         <step name="Test-Map-Put" requires="^"
               exec="onos-execute-expect ${OCI} map-test foo put a b --expect null"/>
 
         <step name="Test-Map-Updated-After-Put" requires="^"
-              exec="onos-execute-expect ${OCI} map-test foo get a --expect b"/>
+              exec="onos-cluster-execute-expect map-test foo get a --expect b"/>
 
         <step name="Test-Map-PutIfAbsent-When-Key-Present" requires="^"
               exec="onos-execute-expect ${OCI} map-test foo putIfAbsent a c --expect b"/>
@@ -40,7 +40,7 @@
               exec="onos-execute-expect ${OCI} map-test foo putIfAbsent b c --expect null"/>
 
         <step name="Test-Map-Updated-After-PutIfAbsent" requires="^"
-              exec="onos-execute-expect ${OCI} map-test foo get b --expect c"/>
+              exec="onos-cluster-execute-expect map-test foo get b --expect c"/>
 
         <step name="Test-Map-Updated-After-PutAndGet" requires="^"
               exec="onos-execute-expect ${OCI} map-test foo putAndGet b d --expect d"/>
@@ -58,16 +58,16 @@
               exec="onos-execute-expect ${OCI} map-test foo replace b e f --expect true"/>
 
         <step name="Test-Map-ContainsValue-False-Case" requires="^"
-              exec="onos-execute-expect ${OCI} map-test foo containsValue x --expect false"/>
+              exec="onos-cluster-execute-expect map-test foo containsValue x --expect false"/>
 
         <step name="Test-Map-ContainsValue-True-Case" requires="^"
-              exec="onos-execute-expect ${OCI} map-test foo containsValue f --expect true"/>
+              exec="onos-cluster-execute-expect map-test foo containsValue f --expect true"/>
 
         <step name="Test-Map-Size" requires="^"
-              exec="onos-execute-expect ${OCI} map-test foo size --expect 2"/>
+              exec="onos-cluster-execute-expect map-test foo size --expect 2"/>
 
         <step name="Test-Map-IsEmpty" requires="^"
-              exec="onos-execute-expect ${OCI} map-test foo isEmpty --expect false"/>
+              exec="onos-cluster-execute-expect map-test foo isEmpty --expect false"/>
 
         <step name="Test-Map-Remove" requires="^"
               exec="onos-execute-expect ${OCI} map-test foo remove b --expect f"/>

@@ -25,25 +25,25 @@
             exec="onos ${OCI} app activate org.onosproject.distributedprimitives"/>
 
         <step name="Test-Value-Initial-Value" requires="^"
-              exec="onos-execute-expect ${OCI} value-test test-value get --expect null"/>
+              exec="onos-cluster-execute-expect value-test test-value get --expect null"/>
 
         <step name="Test-Value-Set" requires="^"
               exec="onos ${OCI} value-test test-value set v0"/>
 
         <step name="Test-Value-Get" requires="^"
-              exec="onos-execute-expect ${OCI} value-test test-value get --expect v0"/>
+              exec="onos-cluster-execute-expect value-test test-value get --expect v0"/>
 
         <step name="Test-Value-CompareAndSet-False" requires="^"
               exec="onos-execute-expect ${OCI} value-test test-value compareAndSet v1 v2 --expect false"/>
 
         <step name="Test-Value-Not-Updated-After-CAS" requires="^"
-              exec="onos-execute-expect ${OCI} value-test test-value get --expect v0"/>
+              exec="onos-cluster-execute-expect value-test test-value get --expect v0"/>
 
         <step name="Test-Value-CompareAndSet-True" requires="^"
               exec="onos-execute-expect ${OCI} value-test test-value compareAndSet v0 v1 --expect true"/>
 
         <step name="Test-Value-Updated-After-CAS" requires="^"
-              exec="onos-execute-expect ${OCI} value-test test-value get --expect v1"/>
+              exec="onos-cluster-execute-expect value-test test-value get --expect v1"/>
 
         <!--Check with check logs-->
         <step name="Check-Log-Exceptions" requires="^"