Option to use rsync pushing bits

Use rsync to push bits (ONOS tar contents), when
environment variable USE_RSYNC is set to true.
This enables sending only changed .jar, now whole ONOS tar ball.

+ Minor optimization to reuse build machine to remote host ssh session

push bits:
 1) unpackage ONOS tar to tmp stage dir
 2) rsync local stage to remote stage /tmp/$ONOS_BITS
 3) package ONOS tar equivalent at remote node

push bits via proxy:
 1) rsync: build <=> proxy
 2)  scp :           proxy => cell nodes (tar packaged at proxy)
 3) rsync: build <==========> cell nodes (almost no-op)

Known limitation:
- rsync at Step 3 is triggered since
  repackaging same content remotely result in
  tar ball with different hash value.
  (Probably due to directory timestamp/uid difference)
- automatically installing rsync
  only supported for debian/ubuntu

Change-Id: Ibc9013b973c312798d2c36e7d25f5274f085b39f
diff --git a/tools/test/bin/onos-push-bits b/tools/test/bin/onos-push-bits
index dff64e0..b65c438 100755
--- a/tools/test/bin/onos-push-bits
+++ b/tools/test/bin/onos-push-bits
@@ -28,12 +28,62 @@
 
 node=${1:-$OCI}
 remote=$ONOS_USER@$node
+SSH_OPTIONS=" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+ -o ControlMaster=auto -o ControlPath=~/.ssh/mux-%r@%h:%p \
+ -o ControlPersist=300 "
 
-locHash=$(cksum $ONOS_TAR | cut -d' ' -f1,2)
-remHash=$(ssh $remote cksum $ONOS_TAR 2>/dev/null | cut -d' ' -f1,2)
+USE_RSYNC=${USE_RSYNC:-'false'}
+if [ "$USE_RSYNC" = "true" ]; then
+  if ssh $remote $SSH_OPTIONS which rsync >&2 > /dev/null; then
+    echo "Using rsync"
+  else
+    echo "Installing rsync"
+    # TODO check remote OS and use proper method to install rsync
+    ssh $remote sudo apt-get install -y rsync || USE_RSYNC='false'
+  fi
+fi
 
-if [ -n "$locHash" ] && [ "$locHash" = "$remHash" ]; then
-    echo "ONOS bits $ONOS_TAR already up-to-date on $node..."
+if [ "$USE_RSYNC" = "clean" ]; then
+  # clean remote rsync stage directory
+  ssh $remote rm -rf "/tmp/$ONOS_BITS"
+fi
+
+if [ "$USE_RSYNC" = "true" ]; then
+  # local rsync stage directory
+  RSYNC_STAGE=`mktemp -d -t onostar` || exit 1
+  trap "rm -rf $RSYNC_STAGE" EXIT
+
+  tar xf $ONOS_TAR -C $RSYNC_STAGE
+  tar tf $ONOS_TAR > $RSYNC_STAGE/files
+  touch -r $ONOS_TAR $RSYNC_STAGE/files
+
+  # initialize remote stage with remote ONOS tar.
+  # This is a workaround to benefit from onos-push-bits-through-proxy.
+  # TODO would like to use tar hash to skip rsync when possible,
+  # but couldn't due to tarball chksum mismatch, described below.
+  ssh $remote $SSH_OPTIONS \
+         "mkdir -p \"/tmp/$ONOS_BITS\" && \
+          tar xzf \"/tmp/`basename ${ONOS_TAR}`\" -C \"/tmp/$ONOS_BITS\" || exit 0 "
+
+  # sync contents of $ONOS_TAR
+  rsync -az --delete --checksum --omit-dir-times --progress \
+        -e "ssh $SSH_OPTIONS" \
+        --rsync-path="mkdir -p /tmp/$ONOS_BITS/ && rsync" \
+        $RSYNC_STAGE/ $remote:/tmp/$ONOS_BITS
+
+  # create $ONOS_TAR equivalent tar ball remotely
+  # TODO hash will not be the same as local one, probably due to different uid, etc.
+  echo "Rebuilding ONOS tar on $node"
+  ssh $remote $SSH_OPTIONS tar czf "/tmp/`basename ${ONOS_TAR}`" -C "/tmp/$ONOS_BITS" --no-recursion -T "/tmp/$ONOS_BITS/files"
 else
-    scp -q $ONOS_TAR $remote:/tmp
+  echo "Using scp"
+
+  locHash=$(cksum $ONOS_TAR | cut -d' ' -f1,2)
+  remHash=$(ssh $remote cksum $ONOS_TAR 2>/dev/null | cut -d' ' -f1,2)
+
+  if [ -n "$locHash" ] && [ "$locHash" = "$remHash" ]; then
+      echo "ONOS bits $ONOS_TAR already up-to-date on $node..."
+  else
+      scp -q $ONOS_TAR $remote:/tmp
+  fi
 fi