Option to use rsync pushing bits
Use rsync to push bits (ONOS tar contents), when
environment variable USE_RSYNC is set to true.
This enables sending only changed .jar, now whole ONOS tar ball.
+ Minor optimization to reuse build machine to remote host ssh session
push bits:
1) unpackage ONOS tar to tmp stage dir
2) rsync local stage to remote stage /tmp/$ONOS_BITS
3) package ONOS tar equivalent at remote node
push bits via proxy:
1) rsync: build <=> proxy
2) scp : proxy => cell nodes (tar packaged at proxy)
3) rsync: build <==========> cell nodes (almost no-op)
Known limitation:
- rsync at Step 3 is triggered since
repackaging same content remotely result in
tar ball with different hash value.
(Probably due to directory timestamp/uid difference)
- automatically installing rsync
only supported for debian/ubuntu
Change-Id: Ibc9013b973c312798d2c36e7d25f5274f085b39f
diff --git a/tools/test/bin/onos-push-bits b/tools/test/bin/onos-push-bits
index dff64e0..b65c438 100755
--- a/tools/test/bin/onos-push-bits
+++ b/tools/test/bin/onos-push-bits
@@ -28,12 +28,62 @@
node=${1:-$OCI}
remote=$ONOS_USER@$node
+SSH_OPTIONS=" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+ -o ControlMaster=auto -o ControlPath=~/.ssh/mux-%r@%h:%p \
+ -o ControlPersist=300 "
-locHash=$(cksum $ONOS_TAR | cut -d' ' -f1,2)
-remHash=$(ssh $remote cksum $ONOS_TAR 2>/dev/null | cut -d' ' -f1,2)
+USE_RSYNC=${USE_RSYNC:-'false'}
+if [ "$USE_RSYNC" = "true" ]; then
+ if ssh $remote $SSH_OPTIONS which rsync >&2 > /dev/null; then
+ echo "Using rsync"
+ else
+ echo "Installing rsync"
+ # TODO check remote OS and use proper method to install rsync
+ ssh $remote sudo apt-get install -y rsync || USE_RSYNC='false'
+ fi
+fi
-if [ -n "$locHash" ] && [ "$locHash" = "$remHash" ]; then
- echo "ONOS bits $ONOS_TAR already up-to-date on $node..."
+if [ "$USE_RSYNC" = "clean" ]; then
+ # clean remote rsync stage directory
+ ssh $remote rm -rf "/tmp/$ONOS_BITS"
+fi
+
+if [ "$USE_RSYNC" = "true" ]; then
+ # local rsync stage directory
+ RSYNC_STAGE=`mktemp -d -t onostar` || exit 1
+ trap "rm -rf $RSYNC_STAGE" EXIT
+
+ tar xf $ONOS_TAR -C $RSYNC_STAGE
+ tar tf $ONOS_TAR > $RSYNC_STAGE/files
+ touch -r $ONOS_TAR $RSYNC_STAGE/files
+
+ # initialize remote stage with remote ONOS tar.
+ # This is a workaround to benefit from onos-push-bits-through-proxy.
+ # TODO would like to use tar hash to skip rsync when possible,
+ # but couldn't due to tarball chksum mismatch, described below.
+ ssh $remote $SSH_OPTIONS \
+ "mkdir -p \"/tmp/$ONOS_BITS\" && \
+ tar xzf \"/tmp/`basename ${ONOS_TAR}`\" -C \"/tmp/$ONOS_BITS\" || exit 0 "
+
+ # sync contents of $ONOS_TAR
+ rsync -az --delete --checksum --omit-dir-times --progress \
+ -e "ssh $SSH_OPTIONS" \
+ --rsync-path="mkdir -p /tmp/$ONOS_BITS/ && rsync" \
+ $RSYNC_STAGE/ $remote:/tmp/$ONOS_BITS
+
+ # create $ONOS_TAR equivalent tar ball remotely
+ # TODO hash will not be the same as local one, probably due to different uid, etc.
+ echo "Rebuilding ONOS tar on $node"
+ ssh $remote $SSH_OPTIONS tar czf "/tmp/`basename ${ONOS_TAR}`" -C "/tmp/$ONOS_BITS" --no-recursion -T "/tmp/$ONOS_BITS/files"
else
- scp -q $ONOS_TAR $remote:/tmp
+ echo "Using scp"
+
+ locHash=$(cksum $ONOS_TAR | cut -d' ' -f1,2)
+ remHash=$(ssh $remote cksum $ONOS_TAR 2>/dev/null | cut -d' ' -f1,2)
+
+ if [ -n "$locHash" ] && [ "$locHash" = "$remHash" ]; then
+ echo "ONOS bits $ONOS_TAR already up-to-date on $node..."
+ else
+ scp -q $ONOS_TAR $remote:/tmp
+ fi
fi
diff --git a/tools/test/bin/onos-push-bits-through-proxy b/tools/test/bin/onos-push-bits-through-proxy
index 21cbddf..6b909f6 100755
--- a/tools/test/bin/onos-push-bits-through-proxy
+++ b/tools/test/bin/onos-push-bits-through-proxy
@@ -16,7 +16,9 @@
others=$(env | sort | egrep "^OC[0-9]+" | cut -d= -f2 | grep -vE "^$OCT\$")
-for other in $others; do
- echo "Pushing to $other..."
- ssh $remote "scp $ONOS_TAR $ONOS_USER@$other:$ONOS_TAR"
-done
+ssh $remote "
+ for other in ${others//$'\n'/ } ; do
+ echo \"Pushing to \$other ...\";
+ scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $ONOS_TAR $ONOS_USER@\$other:$ONOS_TAR
+ done
+"