Update install-p4-tools.sh to use P4Runtime v1.0

Includes also various improvements to the p4vm scripts, such as:
- Choose to build VM with Ubuntu 16.04 or 18.04 (experimental)
- Allow users to specify version of protobuf and grpc to use
- Avoid building protobuf and grpc if already installed in the system
- Install Bazel 0.22

Also:
- Remove check for invalid group member weights (now PI always returns
members with weight 1)
- Re-compiled all P4 programs and fixed missing padding in controller
packet headers

Change-Id: I0e672fcebbaba63354c749f0c774af251f3cbc6c
diff --git a/tools/dev/p4vm/README.md b/tools/dev/p4vm/README.md
index b55793e..a5dfb70 100644
--- a/tools/dev/p4vm/README.md
+++ b/tools/dev/p4vm/README.md
@@ -110,7 +110,8 @@
 
 ### Building the tutorial VM
 
-To build the tutorial VM, simply set the environment variable `P4_VM_TYPE` to `tutorial` before building.
+To build the tutorial VM, simply set the environment variable `P4_VM_TYPE` to
+`tutorial` before building.
 
 For example:
 
@@ -124,3 +125,12 @@
 P4_VM_TYPE=tutorial ./export-ova.sh
 ```
 
+### Support for Ubuntu 18.04
+
+We provide **experimental** support for Ubuntu 18.04 for both the dev and the
+tutorial VM. To specify the Ubuntu version to use, set the environment
+variable `P4_VM_UBUNTU_VERSION` before building. For example:
+
+```bash
+P4_VM_UBUNTU_VERSION=18.04 ./export-ova.sh
+```
diff --git a/tools/dev/p4vm/Vagrantfile b/tools/dev/p4vm/Vagrantfile
index cd4c8c3..a7bb9bf 100644
--- a/tools/dev/p4vm/Vagrantfile
+++ b/tools/dev/p4vm/Vagrantfile
@@ -1,13 +1,14 @@
 P4_VM_TYPE = ENV['P4_VM_TYPE'] || "dev"
+P4_VM_UBUNTU_VERSION = ENV['P4_VM_UBUNTU_VERSION'] || "16.04"
 
 Vagrant.configure(2) do |config|
   if P4_VM_TYPE == "tutorial"
-    config.vm.box = "lasp/ubuntu16.04-desktop"
+    config.vm.box = "lasp/ubuntu" + P4_VM_UBUNTU_VERSION + "-desktop"
   else
-    config.vm.box = "bento/ubuntu-16.04"
+    config.vm.box = "bento/ubuntu-" + P4_VM_UBUNTU_VERSION
   end
   config.vm.provider "virtualbox" do |vb|
-    vb.name = "ONOS+P4 " + P4_VM_TYPE + " " + Time.now.strftime("(%Y-%m-%d)")
+    vb.name = "ONOS+P4 " + P4_VM_TYPE + " " + P4_VM_UBUNTU_VERSION + " " + Time.now.strftime("(%Y-%m-%d)")
     vb.gui = true
     vb.cpus = P4_VM_TYPE == "tutorial" ? 4 : 2
     vb.memory = 4096
diff --git a/tools/dev/p4vm/export-ova.sh b/tools/dev/p4vm/export-ova.sh
index 81f6d03..bc0e5c4 100755
--- a/tools/dev/p4vm/export-ova.sh
+++ b/tools/dev/p4vm/export-ova.sh
@@ -27,6 +27,7 @@
 # Remove references to the existing vagrant-built VM (if any).
 # We want to build a new one from scratch, not start an existing one.
 rm -rf .vagrant/
+vagrant box update
 vagrant up
 
 SSH_PORT=`vagrant port --guest 22`
diff --git a/tools/dev/p4vm/install-p4-tools.sh b/tools/dev/p4vm/install-p4-tools.sh
index 774e007..8b52c8b 100755
--- a/tools/dev/p4vm/install-p4-tools.sh
+++ b/tools/dev/p4vm/install-p4-tools.sh
@@ -17,19 +17,18 @@
 
 # Exit on errors.
 set -e
+set -x
+
+BMV2_COMMIT="d4340832121be1be3852ca0bef709f6443ef86ed"
+PI_COMMIT="81b7e84bf8c27ce87571f66e5ccc76ce228caa8c"
+P4C_COMMIT="5ae390430bd025b301854cd04c78b1ff9902180f"
+
+# p4c seems to break when using protobuf versions newer than 3.2.0
+PROTOBUF_VER=${PROTOBUF_VER:-3.2.0}
+GRPC_VER=${GRPC_VER:-1.3.2}
+
 
 BUILD_DIR=~/p4tools
-# in case BMV2_COMMIT value is updated, the same variable in
-# protocols/bmv2/thrift-api/BUCK file should also be updated
-BMV2_COMMIT="9c8ab62d5680044b94aa2e37b6989f386cc7ae7c"
-PI_COMMIT="a95222eca9b039f6398c048d7e1a1bf7f49b7235"
-P4C_COMMIT="264da2c524c849df0d9ba478cdd1d61b29d64722"
-PROTOBUF_COMMIT="tags/v3.2.0"
-GRPC_COMMIT="tags/v1.3.2"
-LIBYANG_COMMIT="v0.14-r1"
-SYSREPO_COMMIT="v0.7.2"
-
-set -x
 NUM_CORES=`grep -c ^processor /proc/cpuinfo`
 # If false, build tools without debug features to improve throughput of BMv2 and
 # reduce CPU/memory footprint. Default is true.
@@ -88,7 +87,6 @@
         libtool \
         make \
         pkg-config \
-        protobuf-c-compiler \
         python2.7 \
         python2.7-dev \
         tcpdump \
@@ -104,7 +102,6 @@
         ca-certificates \
         g++ \
         libboost-iostreams1.58-dev \
-        libprotobuf-c-dev \
         libreadline6 \
         libreadline6-dev \
         mktemp
@@ -119,41 +116,49 @@
         libboost-regex1.65-dev \
         libboost-iostreams1.65-dev \
         libreadline-dev \
-        libssl1.0-dev \
-        libprotobuf-c-dev
+        libssl1.0-dev
 }
 
 function do_protobuf {
     cd ${BUILD_DIR}
-    if [[ ! -d protobuf ]]; then
-      git clone https://github.com/google/protobuf.git
+    if [[ ! -d protobuf-${PROTOBUF_VER} ]]; then
+      # Get python package which also includes cpp.
+      wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VER}/protobuf-python-${PROTOBUF_VER}.tar.gz
+      tar -xzf protobuf-python-${PROTOBUF_VER}.tar.gz
+      rm -r protobuf-python-${PROTOBUF_VER}.tar.gz
     fi
-    cd protobuf
-    git fetch
-    git checkout ${PROTOBUF_COMMIT}
+    cd protobuf-${PROTOBUF_VER}
 
     export CFLAGS="-Os"
     export CXXFLAGS="-Os"
     export LDFLAGS="-Wl,-s"
     ./autogen.sh
-    ./configure --prefix=/usr
+    confOpts="--prefix=/usr"
+    if [[ "${FAST_BUILD}" = true ]] ; then
+        confOpts="${confOpts} --disable-dependency-tracking"
+    fi
+    ./configure ${confOpts}
     make -j${NUM_CORES}
     sudo make install
     sudo ldconfig
     unset CFLAGS CXXFLAGS LDFLAGS
 
     cd python
-    sudo python setup.py install --cpp_implementation
+    # Hack to get the -std=c++11 flag when building 3.6.1
+    # https://github.com/protocolbuffers/protobuf/blob/v3.6.1/python/setup.py#L208
+    export KOKORO_BUILD_NUMBER="hack"
+    sudo -E python setup.py build --cpp_implementation
+    sudo -E pip install .
+    unset KOKORO_BUILD_NUMBER
 }
 
 function do_grpc {
     cd ${BUILD_DIR}
-    if [[ ! -d grpc ]]; then
-      git clone https://github.com/grpc/grpc.git
+    if [[ ! -d grpc-${GRPC_VER} ]]; then
+      git clone --depth 1 --single-branch --branch v${GRPC_VER} https://github.com/grpc/grpc.git grpc-${GRPC_VER}
     fi
-    cd grpc
-    git fetch
-    git checkout ${GRPC_COMMIT}
+
+    cd grpc-${GRPC_VER}
     git submodule update --init
 
     export LDFLAGS="-Wl,-s"
@@ -189,7 +194,6 @@
 function do_pi_bmv2_deps {
     checkout_bmv2
     # From bmv2's install_deps.sh.
-    # Nanomsg is required also by PI.
     tmpdir=`mktemp -d -p .`
     cd ${tmpdir}
     if [[ "${USE_STRATUM}" = false ]] ; then
@@ -249,7 +253,7 @@
 
     if [[ "${USE_STRATUM}" = false ]] ; then
         # Simple_switch_grpc target (not using Stratum)
-        cd targets/simple_switch_grpc
+        cd ../simple_switch_grpc
         ./autogen.sh
         ./configure --with-thrift
         make -j${NUM_CORES}
@@ -326,6 +330,35 @@
     return 0 # true
 }
 
+function missing_protoc {
+    command -v protoc >/dev/null 2>&1
+    if [[ $? == 0 ]]; then
+        protoc --version | grep $1 &> /dev/null
+        if [[ $? == 0 ]]; then
+            echo "protoc ${1} found!"
+        else
+            echo "A version of protoc was found, but not $1 (you may experience issues)"
+        fi
+        return 1 # false
+    fi
+    return 0 # true
+}
+
+function missing_grpc {
+    # Is there a better way to check if a specific version of grpc is installed?
+    if [[ -f /usr/local/lib/libgrpc++.so ]]; then
+        ls -l /usr/local/lib/libgrpc++.so | grep libgrpc++.so.${1} &> /dev/null
+        if [[ $? == 0 ]]; then
+            echo "grpc ${1} found!"
+        else
+            echo "A version of grpc was found, but not $1 (you may experience issues)"
+        fi
+        return 1 # false
+    else
+        return 0 # true
+    fi
+}
+
 function all_done {
     if [[ "${CLEAN_UP}" = true ]] ; then
         echo "Cleaning up build dir... ${BUILD_DIR})"
@@ -389,11 +422,11 @@
 mkdir -p ${BUILD_DIR}
 cd ${BUILD_DIR}
 # In dependency order.
-if missing_lib libprotobuf; then
-    check_and_do ${PROTOBUF_COMMIT} protobuf do_protobuf protobuf
+if missing_protoc ${PROTOBUF_VER}; then
+    check_and_do ${PROTOBUF_VER} protobuf-${PROTOBUF_VER} do_protobuf protobuf
 fi
-if missing_lib libgrpc; then
-    check_and_do ${GRPC_COMMIT} grpc do_grpc grpc
+if missing_grpc ${GRPC_VER}; then
+    check_and_do ${GRPC_VER} grpc-${GRPC_VER} do_grpc grpc
 fi
 check_and_do ${BMV2_COMMIT} bmv2 do_pi_bmv2_deps bmv2-deps
 check_and_do ${PI_COMMIT} PI do_PI PI
diff --git a/tools/dev/p4vm/pre-ova-cleanup.sh b/tools/dev/p4vm/pre-ova-cleanup.sh
index 45c6a41..754691c 100755
--- a/tools/dev/p4vm/pre-ova-cleanup.sh
+++ b/tools/dev/p4vm/pre-ova-cleanup.sh
@@ -6,13 +6,11 @@
 sudo userdel -r -f vagrant
 
 # Free space on disk
-sudo rm -rf ~/p4tools/protobuf
-sudo rm -rf ~/p4tools/grpc
+sudo rm -rf ~/p4tools/protobuf*
+sudo rm -rf ~/p4tools/grpc*
 sudo rm -rf ~/p4tools/bmv2
 sudo rm -rf ~/p4tools/PI
 sudo rm -rf ~/p4tools/p4c
-sudo rm -rf ~/p4tools/libyang
-sudo rm -rf ~/p4tools/sysrepo
 sudo rm -rf ~/p4tools/scapy-vxlan
 sudo rm -rf ~/p4tools/ptf
 sudo rm -rf ~/quagga
diff --git a/tools/dev/p4vm/root-bootstrap.sh b/tools/dev/p4vm/root-bootstrap.sh
index 0c3302c..90fb557 100755
--- a/tools/dev/p4vm/root-bootstrap.sh
+++ b/tools/dev/p4vm/root-bootstrap.sh
@@ -3,7 +3,7 @@
 
 ONOS_BRANCH_DEV="master"
 ONOS_BRANCH_TUTORIAL="onos-1.14"
-BAZEL_VER="0.19.2"
+BAZEL_VER="0.22.0"
 
 VM_TYPE=${1:-dev}
 
diff --git a/tools/dev/p4vm/user-bootstrap.sh b/tools/dev/p4vm/user-bootstrap.sh
index 6420998..5620506 100755
--- a/tools/dev/p4vm/user-bootstrap.sh
+++ b/tools/dev/p4vm/user-bootstrap.sh
@@ -19,7 +19,7 @@
 EOF
 
 # Build and install P4 tools
-bash /vagrant/install-p4-tools.sh
+DEBUG_FLAGS=true FAST_BUILD=true bash /vagrant/install-p4-tools.sh
 # We'll delete bmv2 sources later...
 cp ~/p4tools/bmv2/tools/veth_setup.sh ~/veth_setup.sh
 cp ~/p4tools/bmv2/tools/veth_teardown.sh ~/veth_teardown.sh
@@ -53,7 +53,3 @@
 
 # Set Python path for bmv2 in fabric.p4
 echo 'export PYTHONPATH=$PYTHONPATH:$ONOS_ROOT/tools/dev/mininet' >> ~/.bashrc
-
-# FIXME: for some reason protobuf python bindings are not properly installed
-cd ~/p4tools/protobuf/python
-sudo pip install .
diff --git a/tools/test/bin/stratum-install b/tools/test/bin/stratum-install
index 72622cd..0008252 100755
--- a/tools/test/bin/stratum-install
+++ b/tools/test/bin/stratum-install
@@ -3,6 +3,8 @@
 # Builds dependencies and copies stratum_bmv2 binary to a remote node.
 # -----------------------------------------------------------------------------
 
+set -e
+
 function _usage () {
 cat << _EOF_
 usage: