Add initial packer definitions

Create initial packer definitions for current bootstrapped environment

Change-Id: I7825cfc993a6156f6c803d63b24e7765a62a5f54
Signed-off-by: Andrew Grimberg <agrimberg@linuxfoundation.org>
diff --git a/packer/README.markdown b/packer/README.markdown
new file mode 100644
index 0000000..33277c1
--- /dev/null
+++ b/packer/README.markdown
@@ -0,0 +1,36 @@
+# EdgeX CI Packer
+
+[Packer][1] is a tool for automatically creating VM and container images,
+configuring them and post-processing them into standard output formats.
+
+We build the CI images via Packer.
+
+## Building
+
+You'll need to [install Packer][2], of course.
+
+The Packer configuration is divided into build-specific variables,
+output-specific templates and a set of shared provisioning scripts. To do a
+specific build, combine the template for the desired output artifact type with
+a variable file. To build a new baseline instance the following would be done:
+
+```
+packer build -var-file=vars/cloud-env.json -var-file=vars/centos.json templates/baseline.json
+```
+
+**NOTE:** vars/cloud-env.json is a gitignored file as it contains private
+information. There is a vars/cloud-env.json.example file that may be used as a
+base for creating the one needed.
+
+From a high level, the build process is:
+
+* Boot a specified base image in any defined clouds.
+* Run a set of shell scripts, listed in the template's shell provisioner
+  section, to do any configuration required by the builder.
+* Execute a shutdown of the running instance in the clouds.
+* Execute the cloud specific method for creating a new image from the shutdown
+  instance
+* Perform the cloud specific method for deleting the shutdown instance
+
+[1]: https://www.packer.io/
+[2]: https://www.packer.io/intro/getting-started/setup.html
diff --git a/packer/provision/basebuild.sh b/packer/provision/basebuild.sh
new file mode 100644
index 0000000..0fd43ff
--- /dev/null
+++ b/packer/provision/basebuild.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# vim: ts=4 sw=4 sts=4 et tw=72 :
+
+# force any errors to cause the script and job to end in failure
+set -xeu -o pipefail
+
+rh_systems() {
+    echo 'No changes to apply'
+}
+
+ubuntu_systems() {
+    apt-get clean
+    apt-get update
+    echo 'No changes to apply'
+}
+
+all_systems() {
+    echo 'No common distribution configuration to perform'
+}
+
+echo "---> Detecting OS"
+ORIGIN=$(facter operatingsystem | tr '[:upper:]' '[:lower:]')
+
+case "${ORIGIN}" in
+    fedora|centos|redhat)
+        echo "---> RH type system detected"
+        rh_systems
+    ;;
+    ubuntu)
+        echo "---> Ubuntu system detected"
+        ubuntu_systems
+    ;;
+    *)
+        echo "---> Unknown operating system"
+    ;;
+esac
+
+# execute steps for all systems
+all_systems
diff --git a/packer/provision/basebuild/.dummy b/packer/provision/basebuild/.dummy
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/packer/provision/basebuild/.dummy
diff --git a/packer/provision/baseline.sh b/packer/provision/baseline.sh
new file mode 100644
index 0000000..6a229b0
--- /dev/null
+++ b/packer/provision/baseline.sh
@@ -0,0 +1,326 @@
+#!/bin/bash
+
+# vim: ts=4 sw=4 sts=4 et tw=72 :
+
+# force any errors to cause the script and job to end in failure
+set -xeu -o pipefail
+
+ensure_kernel_install() {
+    # Workaround for mkinitrd failing on occassion.
+    # On CentOS 7 it seems like the kernel install can fail it's mkinitrd
+    # run quietly, so we may not notice the failure. This script retries for a
+    # few times before giving up.
+    initramfs_ver=$(rpm -q kernel | tail -1 | sed "s/kernel-/initramfs-/")
+    grub_conf="/boot/grub/grub.conf"
+    # Public cloud does not use /boot/grub/grub.conf and uses grub2 instead.
+    if [ ! -e "$grub_conf" ]; then
+        echo "$grub_conf not found. Using Grub 2 conf instead."
+        grub_conf="/boot/grub2/grub.cfg"
+    fi
+
+    for i in $(seq 3); do
+        if grep "$initramfs_ver" "$grub_conf"; then
+            break
+        fi
+        echo "Kernel initrd missing. Retrying to install kernel..."
+        yum reinstall -y kernel
+    done
+    if ! grep "$initramfs_ver" "$grub_conf"; then
+        cat /boot/grub/grub.conf
+        echo "ERROR: Failed to install kernel."
+        exit 1
+    fi
+}
+
+ensure_ubuntu_install() {
+    # Workaround for mirrors occassionally failing to install a package.
+    # On Ubuntu sometimes the mirrors fail to install a package. This wrapper
+    # checks that a package is successfully installed before moving on.
+
+    packages=($@)
+
+    for pkg in "${packages[@]}"
+    do
+        # Retry installing package 5 times if necessary
+        for i in {0..5}
+        do
+            if [ "$(dpkg-query -W -f='${Status}' "$pkg" 2>/dev/null | grep -c "ok installed")" -eq 0 ]; then
+                apt-cache policy "$pkg"
+                apt-get install "$pkg"
+                continue
+            else
+                echo "$pkg already installed."
+                break
+            fi
+        done
+    done
+}
+
+rh_systems() {
+    # Handle the occurance where SELINUX is actually disabled
+    SELINUX=$(grep -E '^SELINUX=(disabled|permissive|enforcing)$' /etc/selinux/config)
+    MODE=$(echo "$SELINUX" | cut -f 2 -d '=')
+    case "$MODE" in
+        permissive)
+            echo "************************************"
+            echo "** SYSTEM ENTERING ENFORCING MODE **"
+            echo "************************************"
+            # make sure that the filesystem is properly labelled.
+            # it could be not fully labeled correctly if it was just switched
+            # from disabled, the autorelabel misses some things
+            # skip relabelling on /dev as it will generally throw errors
+            restorecon -R -e /dev /
+
+            # enable enforcing mode from the very start
+            setenforce enforcing
+
+            # configure system for enforcing mode on next boot
+            sed -i 's/SELINUX=permissive/SELINUX=enforcing/' /etc/selinux/config
+        ;;
+        disabled)
+            sed -i 's/SELINUX=disabled/SELINUX=permissive/' /etc/selinux/config
+            touch /.autorelabel
+
+            echo "*******************************************"
+            echo "** SYSTEM REQUIRES A RESTART FOR SELINUX **"
+            echo "*******************************************"
+        ;;
+        enforcing)
+            echo "*********************************"
+            echo "** SYSTEM IS IN ENFORCING MODE **"
+            echo "*********************************"
+        ;;
+    esac
+
+    # Allow jenkins access to alternatives command to switch java version
+    cat <<EOF >/etc/sudoers.d/89-jenkins-user-defaults
+Defaults:jenkins !requiretty
+jenkins ALL = NOPASSWD: /usr/sbin/alternatives
+EOF
+
+    echo "---> Updating operating system"
+    yum clean all
+    yum install -y deltarpm
+    yum update -y
+
+    ensure_kernel_install
+
+    # add in components we need or want on systems
+    echo "---> Installing base packages"
+    yum install -y @base https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+    # separate group installs from package installs since a non-existing
+    # group with dnf based systems (F21+) will fail the install if such
+    # a group does not exist
+    yum install -y unzip xz puppet git git-review perl-XML-XPath
+    yum install -y python-{devel,virtualenv,setuptools,pip}
+
+    # All of our systems require Java (because of Jenkins)
+    # Install all versions of the OpenJDK devel but force 1.7.0 to be the
+    # default
+
+    echo "---> Configuring OpenJDK"
+    yum install -y 'java-*-openjdk-devel'
+
+    FACTER_OS=$(/usr/bin/facter operatingsystem)
+    FACTER_OSVER=$(/usr/bin/facter operatingsystemrelease)
+    case "$FACTER_OS" in
+        Fedora)
+            if [ "$FACTER_OSVER" -ge "21" ]
+            then
+                echo "---> not modifying java alternatives as OpenJDK 1.7.0 does not exist"
+            else
+                alternatives --set java /usr/lib/jvm/jre-1.7.0-openjdk.x86_64/bin/java
+                alternatives --set java_sdk_openjdk /usr/lib/jvm/java-1.7.0-openjdk.x86_64
+            fi
+        ;;
+        RedHat|CentOS)
+            if [ "$(echo "$FACTER_OSVER" | cut -d'.' -f1)" -ge "7" ]
+            then
+                echo "---> not modifying java alternatives as OpenJDK 1.7.0 does not exist"
+            else
+                alternatives --set java /usr/lib/jvm/jre-1.7.0-openjdk.x86_64/bin/java
+                alternatives --set java_sdk_openjdk /usr/lib/jvm/java-1.7.0-openjdk.x86_64
+            fi
+        ;;
+        *)
+            alternatives --set java /usr/lib/jvm/jre-1.7.0-openjdk.x86_64/bin/java
+            alternatives --set java_sdk_openjdk /usr/lib/jvm/java-1.7.0-openjdk.x86_64
+        ;;
+    esac
+
+    ########################
+    # --- START LFTOOLS DEPS
+
+    # Used by various scripts to push patches to Gerrit
+    yum install -y git-review
+
+    # Needed to parse OpenStack commands used by opendaylight-infra stack commands
+    # to initialize Heat template based systems.
+    yum install -y jq
+
+    # Used by lftools scripts to parse XML
+    yum install -y xmlstarlet
+
+    # Haskel Packages
+    # Cabal update fails on a 1G system so workaround that with a swap file
+    dd if=/dev/zero of=/tmp/swap bs=1M count=1024
+    mkswap /tmp/swap
+    swapon /tmp/swap
+
+    yum install -y cabal-install
+    cabal update
+    cabal install "Cabal<1.18"  # Pull Cabal version that is capable of building shellcheck
+    cabal install --bindir=/usr/local/bin "shellcheck-0.4.6"  # Pin shellcheck version
+
+    # --- END LFTOOLS DEPS
+    ######################
+
+    # install haveged to avoid low entropy rejecting ssh connections
+    yum install -y haveged
+    systemctl enable haveged.service
+}
+
+ubuntu_systems() {
+    # Ignore SELinux since slamming that onto Ubuntu leads to
+    # frustration
+
+    # Allow jenkins access to update-alternatives command to switch java version
+    cat <<EOF >/etc/sudoers.d/89-jenkins-user-defaults
+Defaults:jenkins !requiretty
+jenkins ALL = NOPASSWD: /usr/bin/update-alternatives
+EOF
+
+    export DEBIAN_FRONTEND=noninteractive
+    cat <<EOF >> /etc/apt/apt.conf
+APT {
+  Get {
+    Assume-Yes "true";
+    allow-change-held-packages "true";
+    allow-downgrades "true";
+    allow-remove-essential "true";
+  };
+};
+
+Dpkg::Options {
+  "--force-confdef";
+  "--force-confold";
+};
+
+EOF
+
+    # Add hostname to /etc/hosts to fix 'unable to resolve host' issue with sudo
+    sed -i "/127.0.0.1/s/$/ $(hostname)/" /etc/hosts
+
+    echo "---> Updating operating system"
+
+    # add additional repositories
+    sudo add-apt-repository "deb http://us.archive.ubuntu.com/ubuntu $(lsb_release -sc) main universe restricted multiverse"
+
+    echo "---> Installing base packages"
+    apt-get clean
+    apt-get update -m
+    apt-get upgrade -m
+    apt-get dist-upgrade -m
+
+    ensure_ubuntu_install unzip xz-utils puppet git libxml-xpath-perl
+
+    # install Java 7
+    echo "---> Configuring OpenJDK"
+    FACTER_OSVER=$(/usr/bin/facter operatingsystemrelease)
+    case "$FACTER_OSVER" in
+        14.04)
+            apt-get install openjdk-7-jdk
+            # make jdk8 available
+            add-apt-repository -y ppa:openjdk-r/ppa
+            apt-get update
+            # We need to force openjdk-8-jdk to install
+            apt-get install openjdk-8-jdk
+            # make sure that we still default to openjdk 7
+            update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java
+            update-alternatives --set javac /usr/lib/jvm/java-7-openjdk-amd64/bin/javac
+        ;;
+        16.04)
+            apt-get install openjdk-8-jdk
+        ;;
+        *)
+            echo "---> Unknown Ubuntu version $FACTER_OSVER"
+            exit 1
+        ;;
+    esac
+
+    ########################
+    # --- START LFTOOLS DEPS
+
+    # Used by various scripts to push patches to Gerrit
+    ensure_ubuntu_install git-review
+
+    # Needed to parse OpenStack commands used by opendaylight-infra stack commands
+    # to initialize Heat template based systems.
+    ensure_ubuntu_install jq
+
+    # Used by lftools scripts to parse XML
+    ensure_ubuntu_install xmlstarlet
+
+    # Haskel Packages
+    # Cabal update fails on a 1G system so workaround that with a swap file
+    dd if=/dev/zero of=/tmp/swap bs=1M count=1024
+    mkswap /tmp/swap
+    swapon /tmp/swap
+
+    ensure_ubuntu_install cabal-install
+    cabal update
+    cabal install --bindir=/usr/local/bin "shellcheck-0.4.6"  # Pin shellcheck version
+
+    # --- END LFTOOLS DEPS
+    ######################
+
+    # install haveged to avoid low entropy rejecting ssh connections
+    apt-get install haveged
+    update-rc.d haveged defaults
+
+    # disable unattended upgrades & daily updates
+    echo '---> Disabling automatic daily upgrades'
+    sed -ine 's/"1"/"0"/g' /etc/apt/apt.conf.d/10periodic
+    echo 'APT::Periodic::Unattended-Upgrade "0";' >> /etc/apt/apt.conf.d/10periodic
+}
+
+all_systems() {
+    # Do any Distro specific installations here
+    echo "Checking distribution"
+    FACTER_OS=$(/usr/bin/facter operatingsystem)
+    case "$FACTER_OS" in
+        *)
+            echo "---> $FACTER_OS found"
+            echo "No extra steps for $FACTER_OS"
+        ;;
+    esac
+}
+
+echo "---> Attempting to detect OS"
+# upstream cloud images use the distro name as the initial user
+ORIGIN=$(if [ -e /etc/redhat-release ]
+    then
+        echo redhat
+    else
+        echo ubuntu
+    fi)
+#ORIGIN=$(logname)
+
+case "${ORIGIN}" in
+    fedora|centos|redhat)
+        echo "---> RH type system detected"
+        rh_systems
+    ;;
+    ubuntu)
+        echo "---> Ubuntu system detected"
+        ubuntu_systems
+    ;;
+    *)
+        # Kill the build for unhandled distributions
+        echo "---> Unknown operating system" 1>&2
+        exit 1
+    ;;
+esac
+
+# execute steps for all systems
+all_systems
diff --git a/packer/provision/null_data.sh b/packer/provision/null_data.sh
new file mode 100644
index 0000000..3fa6a3c
--- /dev/null
+++ b/packer/provision/null_data.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+# vi: ts=4 sw=4 sts=4 et :
+
+# Nothing to do for Ubuntu specific provisioning
diff --git a/packer/provision/rh-user_data.sh b/packer/provision/rh-user_data.sh
new file mode 100644
index 0000000..6bddb24
--- /dev/null
+++ b/packer/provision/rh-user_data.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+# vi: ts=4 sw=4 sts=4 et :
+
+/bin/sed -i 's/ requiretty/ !requiretty/' /etc/sudoers;
diff --git a/packer/provision/system_reseal.sh b/packer/provision/system_reseal.sh
new file mode 100644
index 0000000..f8bc7dc
--- /dev/null
+++ b/packer/provision/system_reseal.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# vim: sw=2 ts=2 sts=2 et :
+
+rm -rf /etc/Pegasus/*.cnf /etc/Pegasus/*.crt /etc/Pegasus/*.csr \
+  /etc/Pegasus/*.pem /etc/Pegasus/*.srl /root/anaconda-ks.cfg \
+  /root/anaconda-post.log /root/initial-setup-ks.cfg /root/install.log \
+  /root/install.log.syslog /var/cache/fontconfig/* /var/cache/gdm/* \
+  /var/cache/man/* /var/lib/AccountService/users/* /var/lib/fprint/* \
+  /var/lib/logrotate.status /var/log/*.log* /var/log/BackupPC/LOG \
+  /var/log/ConsoleKit/* /var/log/anaconda.syslog /var/log/anaconda/* \
+  /var/log/apache2/*_log /var/log/apache2/*_log-* /var/log/apt/* \
+  /var/log/aptitude* /var/log/audit/* /var/log/btmp* /var/log/ceph/*.log \
+  /var/log/chrony/*.log /var/log/cron* /var/log/cups/*_log /var/log/debug* \
+  /var/log/dmesg* /var/log/exim4/* /var/log/faillog* /var/log/gdm/* \
+  /var/log/glusterfs/*glusterd.vol.log /var/log/glusterfs/glusterfs.log \
+  /var/log/httpd/*log /var/log/installer/* /var/log/jetty/jetty-console.log \
+  /var/log/journal/* /var/log/lastlog* /var/log/libvirt/libvirtd.log \
+  /var/log/libvirt/lxc/*.log /var/log/libvirt/qemu/*.log \
+  /var/log/libvirt/uml/*.log /var/log/lightdm/* /var/log/mail/* \
+  /var/log/maillog* /var/log/messages* /var/log/ntp /var/log/ntpstats/* \
+  /var/log/ppp/connect-errors /var/log/rhsm/* /var/log/sa/* /var/log/secure* \
+  /var/log/setroubleshoot/*.log /var/log/spooler* /var/log/squid/*.log \
+  /var/log/syslog* /var/log/tallylog* /var/log/tuned/tuned.log /var/log/wtmp* \
+  /var/named/data/named.run
+
+rm -rf ~/.viminfo /etc/ssh/ssh*key* /root/.ssh/*
+
+# kill any cloud-init related bits
+rm -rf /var/lib/cloud/*
+
+# clean-up any manual packer uploads
+rm -rf /tmp/packer
+
+# Force a system sync and sleep to get around any SSD issues
+echo "Forcing sync and sleep for 10sec"
+sync
+sleep 10
diff --git a/packer/provision/system_reseal_local_env.sh b/packer/provision/system_reseal_local_env.sh
new file mode 100644
index 0000000..041fffe
--- /dev/null
+++ b/packer/provision/system_reseal_local_env.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+# Here add local env changes.
diff --git a/packer/templates/basebuild.json b/packer/templates/basebuild.json
new file mode 100644
index 0000000..da6f271
--- /dev/null
+++ b/packer/templates/basebuild.json
@@ -0,0 +1,48 @@
+{
+  "variables": {
+    "aws_access_key": null,
+    "aws_security_key": null,
+    "security_group_id": null,
+    "source_ami_filter_name": null,
+    "source_ami_filter_owner": null,
+    "subnet_id": null,
+    "cloud_user": null,
+    "distro": null,
+    "cloud_user_data": null
+  },
+  "builders": [
+    {
+      "access_key": "{{user `aws_access_key`}}",
+      "ami_name": "{{user `distro`}} - basebuild - {{isotime \"20060102-1504\"}}",
+      "instance_type": "t2.micro",
+      "region": "us-west-2",
+      "secret_key": "{{user `aws_security_key`}}",
+      "security_group_id": "{{user `security_group_id`}}",
+      "source_ami_filter": {
+        "filters": {
+          "name": "{{user `source_ami_filter_name`}}",
+          "root-device-type": "ebs",
+          "virtualization-type": "hvm"
+        },
+        "most_recent": true,
+        "owners": ["{{user `source_ami_filter_owner`}}"]
+      },
+      "ssh_username": "{{user `cloud_user`}}",
+      "subnet_id": "{{user `subnet_id`}}",
+      "type": "amazon-ebs",
+      "user_data_file": "{{user `cloud_user_data`}}"
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "shell",
+      "scripts": [
+        "provision/baseline.sh",
+        "provision/basebuild.sh",
+        "provision/system_reseal_local_env.sh",
+        "provision/system_reseal.sh"
+      ],
+      "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi"
+    }
+  ]
+}
diff --git a/packer/templates/baseline.json b/packer/templates/baseline.json
new file mode 100644
index 0000000..532e15a
--- /dev/null
+++ b/packer/templates/baseline.json
@@ -0,0 +1,47 @@
+{
+  "variables": {
+    "aws_access_key": null,
+    "aws_security_key": null,
+    "security_group_id": null,
+    "source_ami_filter_name": null,
+    "source_ami_filter_owner": null,
+    "subnet_id": null,
+    "cloud_user": null,
+    "distro": null,
+    "cloud_user_data": null
+  },
+  "builders": [
+    {
+      "access_key": "{{user `aws_access_key`}}",
+      "ami_name": "{{user `distro`}} - baseline - {{isotime \"20060102-1504\"}}",
+      "instance_type": "t2.micro",
+      "region": "us-west-2",
+      "secret_key": "{{user `aws_security_key`}}",
+      "security_group_id": "{{user `security_group_id`}}",
+      "source_ami_filter": {
+        "filters": {
+          "name": "{{user `source_ami_filter_name`}}",
+          "root-device-type": "ebs",
+          "virtualization-type": "hvm"
+        },
+        "most_recent": true,
+        "owners": ["{{user `source_ami_filter_owner`}}"]
+      },
+      "ssh_username": "{{user `cloud_user`}}",
+      "subnet_id": "{{user `subnet_id`}}",
+      "type": "amazon-ebs",
+      "user_data_file": "{{user `cloud_user_data`}}"
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "shell",
+      "scripts": [
+        "provision/baseline.sh",
+        "provision/system_reseal_local_env.sh",
+        "provision/system_reseal.sh"
+      ],
+      "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi"
+    }
+  ]
+}
diff --git a/packer/vars/.gitignore b/packer/vars/.gitignore
new file mode 100644
index 0000000..af92028
--- /dev/null
+++ b/packer/vars/.gitignore
@@ -0,0 +1,2 @@
+# exclude private cloud-env settings
+cloud-env.json
diff --git a/packer/vars/centos-7.json b/packer/vars/centos-7.json
new file mode 100644
index 0000000..fefd0aa
--- /dev/null
+++ b/packer/vars/centos-7.json
@@ -0,0 +1,8 @@
+{
+  "source_ami_filter_name": "*CentOS Linux 7*HVM*",
+  "source_ami_filter_owner": "679593333241",
+  "cloud_user": "centos",
+
+  "distro": "CentOS 7",
+  "cloud_user_data": "provision/rh-user_data.sh"
+}
diff --git a/packer/vars/cloud-env.json.example b/packer/vars/cloud-env.json.example
new file mode 100644
index 0000000..dc1c08f
--- /dev/null
+++ b/packer/vars/cloud-env.json.example
@@ -0,0 +1,3 @@
+{
+  "subnet_id": "subnet-NNNNNNNN"
+}
diff --git a/packer/vars/ubuntu-16.04.json b/packer/vars/ubuntu-16.04.json
new file mode 100644
index 0000000..370af61
--- /dev/null
+++ b/packer/vars/ubuntu-16.04.json
@@ -0,0 +1,8 @@
+{
+  "source_ami_filter_name": "*Ubuntu*16.04*",
+  "source_ami_filter_owner": "099720109477",
+  "cloud_user": "ubuntu",
+
+  "distro": "Ubuntu 16.04",
+  "cloud_user_data": "provision/null_data.sh"
+}