Added tagging/versioning/publishing job templates
- Uses the CORD-style versioning workflow described here:
https://guide.opencord.org/developer/test_release_software.html#versioning-projects
but with refactoring and renaming for clarification.
- Added trellis-docs test/publish jobs using these templates
- Cleanup of defaults.yaml file
- Added rsync and virtualenv to ubuntu basebuild
Change-Id: I03692460a5b6eee6b8f131f0fbcc9c5957819ddd
diff --git a/.gitignore b/.gitignore
index 11b529f..8a578e0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,4 +29,7 @@
__pycache__/
*.pyc
-onap_sandbox
+# JJB testing related
+venv-jjb
+job-configs
+
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..cfa1cd1
--- /dev/null
+++ b/README.md
@@ -0,0 +1,16 @@
+# ONOS ci-management repo
+
+Sets up the CI system for ONOS on Jenkins via Jenkins Job Builder, and the EC2
+executor nodes via Packer.
+
+## Notable files
+
+- `jjb/defaults.yaml`: default values used to parameterize all jjb jobs
+
+- `jjb/onf-macros.yaml`: JJB macros used by other jobs that set common behavior
+ and reduce verbosity of normal jobs.
+
+- `jjb/templates/*.yaml`: JJB job-templates that are used by other jobs.
+
+- `jjb/repos/*.yaml`: normal location of per-gerrit-repo validation jobs, file
+ name is the same as the repo name.
diff --git a/jjb/defaults.yaml b/jjb/defaults.yaml
index b4babc4..65ade31 100644
--- a/jjb/defaults.yaml
+++ b/jjb/defaults.yaml
@@ -10,33 +10,56 @@
# lf-infra-defaults
jenkins-ssh-credential: 'onos-jenkins-ssh'
- # build discards
- build-days-to-keep: 30
-
# Timeout in minutes
#TODO deprecate this (should be project template specific)
build-timeout: 360
- #TODO this should be the most common executor
- build-node: centos7-basebuild-1c-1g
+
+ # How long to keep builds and artifacts
+ build-days-to-keep: 60
+ artifact-num-to-keep: 30
+
+ # The most frequently used type of build node
+ # see other build node types under "Cloud > Amazon EC2" at
+ # https://jenkins.onosproject.org/configure
+ build-node: ubuntu16.04-basebuild-1c-1g
# default gerrit server definition
#server-name: 'ONOS Project Gerrit'
gerrit-server-name: 'ONOS Project Gerrit'
+ # User account with gerrit SSH credentials
+ gerrit-ssh-credential: 'onos-gerrit-ssh'
+
+ # Java glob of artifacts to archive
archive-artifacts: ''
- # Set default maven version used for everything
- mvn-version: 'mvn33'
+ # by default, don't depend on other jobs
+ dependency-jobs: ''
- # Default maven goals
- mvn-goals: 'clean install'
- mvn-opts: ''
+ # used to rename jobs if required
+ name-extension: ''
- # Maven / Java
- edgex-infra-mvn-opts: |
- --show-version
- --batch-mode
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
- -Djenkins
- -Dmaven.repo.local=/tmp/r
- -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r
+ # regexes for branch matching
+ all-branches-regexp: '.*'
+ supported-branches-regexp: '.*'
+
+ # regexes for file matching
+ all-files-regexp: '.*'
+
+ # strictness of version checks - if set to 1, then only allow SemVer versioning
+ semver-strict: 0
+
+ # Make test targets
+ # List of targets to run when testing a patchset, run with make
+ # defaults to just 'test', multiple targets should be space separated
+ make-test-targets: 'test'
+
+ # whether to "keep going" on multiple tests if one fails
+ # maps to the `-k` option passed to make in make-test.yaml
+ make-test-keep-going: false
+
+ # golang specific variables
+ # dest-gopath handles checking out patchsets and putting them into a GOPATH
+ # This portion of the path should be included: `$GOPATH/src/<dest-gopath>/<project>"
+ # If blank, golang related variables won't be set
+ dest-gopath: ''
diff --git a/jjb/onf-macros.yaml b/jjb/onf-macros.yaml
new file mode 100644
index 0000000..41c85b1
--- /dev/null
+++ b/jjb/onf-macros.yaml
@@ -0,0 +1,102 @@
+# JJB Macros for ONF jobs
+
+# control how long builds and artifact are retained
+# differs from lf-infra-properties as it retains artifacts
+- property:
+ name: onf-infra-properties
+ properties:
+ - build-discarder:
+ days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+
+# trigger on gerrit patchsets and actions
+# docs: https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit
+# Uses a regex based project match
+- trigger:
+ name: onf-infra-gerrit-trigger-patchset
+ triggers:
+ - gerrit:
+ server-name: '{gerrit-server-name}'
+ dependency-jobs: '{dependency-jobs}'
+ silent-start: true
+ trigger-on:
+ - patchset-created-event:
+ exclude-drafts: true
+ exclude-trivial-rebase: false
+ exclude-no-code-change: false
+ - draft-published-event
+ - comment-added-contains-event:
+ comment-contains-value: '(?i)^.*recheck$'
+ projects:
+ - project-compare-type: REG_EXP
+ project-pattern: '{project-regexp}'
+ branches:
+ - branch-compare-type: REG_EXP
+ branch-pattern: '{branch-regexp}'
+ file-paths:
+ - compare-type: REG_EXP
+ pattern: '{file-include-regexp}'
+
+
+# same as lf-infra-gerrit-scm, but allows checkouts to a subdir of $WORKSPACE
+# with the `basedir` option
+#
+# `basedir` serves the same function as `destination-dir` in the repo scm
+# macros, seems strange that they're named differently.
+- scm:
+ name: onf-infra-gerrit-scm
+ scm:
+ - git:
+ credentials-id: '{jenkins-ssh-credential}'
+ url: '{git-url}'
+ refspec: '{refspec}'
+ branches:
+ - 'refs/heads/{branch}'
+ skip-tag: true
+ wipe-workspace: true
+ submodule:
+ recursive: '{submodule-recursive}'
+ choosing-strategy: '{choosing-strategy}'
+ basedir: '{basedir}'
+
+
+# trigger for gerrit patch submission
+- trigger:
+ name: onf-infra-gerrit-trigger-merge
+ triggers:
+ - gerrit:
+ server-name: '{gerrit-server-name}'
+ dependency-jobs: '{dependency-jobs}'
+ silent-start: true
+ trigger-on:
+ - change-merged-event
+ projects:
+ - project-compare-type: REG_EXP
+ project-pattern: '{project-regexp}'
+ branches:
+ - branch-compare-type: REG_EXP
+ branch-pattern: '{branch-regexp}'
+ file-paths:
+ - compare-type: REG_EXP
+ pattern: '{file-include-regexp}'
+
+# wrapper to provide SSH key and fill in ~/.ssh/known_hosts file for use with rsync
+- wrapper:
+ name: onf-infra-rsync-wrappers
+ wrappers:
+ - mask-passwords
+ - timeout:
+ type: absolute
+ timeout: '{build-timeout}'
+ timeout-var: 'BUILD_TIMEOUT'
+ fail: true
+ - timestamps
+ - ssh-agent-credentials:
+ users:
+ - '{jenkins-ssh-credential}'
+ - config-file-provider:
+ files:
+ - file-id: known_hosts
+ target: '$HOME/.ssh/known_hosts'
+
diff --git a/jjb/repos/trellis-docs.yaml b/jjb/repos/trellis-docs.yaml
new file mode 100644
index 0000000..f9e5851
--- /dev/null
+++ b/jjb/repos/trellis-docs.yaml
@@ -0,0 +1,32 @@
+---
+# jobs for 'trellis-docs' repo
+
+- project:
+ name: trellis-docs
+ project: '{name}'
+
+ jobs:
+ - 'verify-trellis-docs-jobs':
+ branch-regexp: '{supported-branches-regexp}'
+ - 'publish-trellis-docs-jobs':
+ branch-regexp: '{supported-branches-regexp}'
+
+- job-group:
+ name: 'verify-trellis-docs-jobs'
+ jobs:
+ - 'verify-licensed'
+ - 'tag-check':
+ dependency-jobs: 'license-check_trellis-docs'
+ - 'make-test':
+ junit-allow-empty-results: true
+
+- job-group:
+ name: 'publish-trellis-docs-jobs'
+ jobs:
+ - 'version-tag'
+ - 'sync-dir':
+ dependency-jobs: 'version-tag_trellis-docs'
+ build-command: 'make html'
+ build-output-path: 'build/html'
+ sync-target-server: 'guide.opencord.org'
+ sync-target-path: '/var/www/trellis-docs/'
diff --git a/jjb/shell/licensecheck.sh b/jjb/shell/licensecheck.sh
new file mode 100755
index 0000000..5bbcdf8
--- /dev/null
+++ b/jjb/shell/licensecheck.sh
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+
+# licensecheck.sh
+# checks for copyright/license headers on files
+# excludes filename extensions where this check isn't pertinent
+
+set +e -u -o pipefail
+fail_licensecheck=0
+
+while IFS= read -r -d '' f
+do
+ grep -q "Copyright\|Apache License" "${f}"
+ rc=$?
+ if [[ $rc != 0 ]]; then
+ echo "ERROR: $f does not contain License Header"
+ fail_licensecheck=1
+ fi
+done < <(find . -name ".git" -prune -o -type f \
+ -name "*.*" \
+ ! -name "*.PNG" \
+ ! -name "*.asc" \
+ ! -name "*.bat" \
+ ! -name "*.cert" \
+ ! -name "*.cfg" \
+ ! -name "*.cnf" \
+ ! -name "*.conf" \
+ ! -name "*.cql" \
+ ! -name "*.crt" \
+ ! -name "*.csar" \
+ ! -name "*.csr" \
+ ! -name "*.csv" \
+ ! -name "*.ctmpl" \
+ ! -name "*.curl" \
+ ! -name "*.db" \
+ ! -name "*.der" \
+ ! -name "*.desc" \
+ ! -name "*.diff" \
+ ! -name "*.dnsmasq" \
+ ! -name "*.do" \
+ ! -name "*.docx" \
+ ! -name "*.eot" \
+ ! -name "*.gif" \
+ ! -name "*.gpg" \
+ ! -name "*.graffle" \
+ ! -name "*.ico" \
+ ! -name "*.iml" \
+ ! -name "*.in" \
+ ! -name "*.inc" \
+ ! -name "*.install" \
+ ! -name "*.j2" \
+ ! -name "*.jar" \
+ ! -name "*.jks" \
+ ! -name "*.jpg" \
+ ! -name "*.json" \
+ ! -name "*.jsonld" \
+ ! -name "*.JSON" \
+ ! -name "*.key" \
+ ! -name "*.list" \
+ ! -name "*.local" \
+ ! -path "*.lock" \
+ ! -name "*.log" \
+ ! -name "*.mak" \
+ ! -name "*.md" \
+ ! -name "*.MF" \
+ ! -name "*.mk" \
+ ! -name "*.oar" \
+ ! -name "*.p12" \
+ ! -name "*.patch" \
+ ! -name "*.pb.go" \
+ ! -name "*.pdf" \
+ ! -name "*.pcap" \
+ ! -name "*.pem" \
+ ! -name "*.png" \
+ ! -name "*.properties" \
+ ! -name "*.proto" \
+ ! -name "*.protoset" \
+ ! -name "*.pyc" \
+ ! -name "*.repo" \
+ ! -name "*.robot" \
+ ! -name "*.rst" \
+ ! -name "*.rules" \
+ ! -name "*.service" \
+ ! -name "*.svg" \
+ ! -name "*.swp" \
+ ! -name "*.tar" \
+ ! -name "*.tar.gz" \
+ ! -name "*.toml" \
+ ! -name "*.ttf" \
+ ! -name "*.txt" \
+ ! -name "*.woff" \
+ ! -name "*.xproto" \
+ ! -name "*.xtarget" \
+ ! -name "*ignore" \
+ ! -name "*rc" \
+ ! -name "*_pb2.py" \
+ ! -name "*_pb2_grpc.py" \
+ ! -name "Dockerfile" \
+ ! -name "Dockerfile.*" \
+ ! -name "go.mod" \
+ ! -name "go.sum" \
+ ! -name "Makefile" \
+ ! -name "Makefile.*" \
+ ! -name "README" \
+ ! -path "*/vendor/*" \
+ ! -path "*conf*" \
+ ! -path "*git*" \
+ ! -path "*swagger*" \
+ -print0 )
+
+exit ${fail_licensecheck}
diff --git a/jjb/shell/make-test.sh b/jjb/shell/make-test.sh
new file mode 100644
index 0000000..b92de49
--- /dev/null
+++ b/jjb/shell/make-test.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# make-test.sh - run one or more make targets
+set -eu -o pipefail
+
+# when not running under Jenkins, use current dir as workspace, a blank project
+# name
+WORKSPACE=${WORKSPACE:-.}
+GERRIT_PROJECT=${GERRIT_PROJECT:-}
+
+# Fixes to for golang projects to support GOPATH
+# If $DEST_GOPATH is not an empty string:
+# - set create GOPATH, and destination directory within in
+# - set PATH to include $GOPATH/bin and the system go binaries
+# - symlink from $WORKSPACE/$GERRIT_PROJECT to new location in $GOPATH
+# - start tests in that directory
+
+DEST_GOPATH=${DEST_GOPATH:-}
+if [ ! -z "$DEST_GOPATH" ]; then
+ export GOPATH=${GOPATH:-~/go}
+ mkdir -p "$GOPATH/src/$DEST_GOPATH"
+ export PATH=$PATH:/usr/lib/go-1.12/bin:/usr/local/go/bin:$GOPATH/bin
+ test_path="$GOPATH/src/$DEST_GOPATH/$GERRIT_PROJECT"
+ ln -r -s "$WORKSPACE/$GERRIT_PROJECT" "$test_path"
+else
+ test_path="$WORKSPACE/$GERRIT_PROJECT"
+fi
+
+# Use "test" as the default target, can be a space separated list
+MAKE_TEST_TARGETS=${MAKE_TEST_TARGETS:-test}
+
+# Default to fail on the first test that fails
+MAKE_TEST_KEEP_GOING=${MAKE_TEST_KEEP_GOING:-false}
+
+if [ ! -f "$test_path/Makefile" ]; then
+ echo "Makefile not found at $test_path!"
+ exit 1
+else
+ pushd "$test_path"
+
+ # we want to split the make targets apart on spaces, so:
+ # shellcheck disable=SC2086
+ if [ "$MAKE_TEST_KEEP_GOING" = "true" ]; then
+ make -k $MAKE_TEST_TARGETS
+ else
+ make $MAKE_TEST_TARGETS
+ fi
+
+ popd
+fi
+
diff --git a/jjb/shell/sync-dir.sh b/jjb/shell/sync-dir.sh
new file mode 100644
index 0000000..d7f4218
--- /dev/null
+++ b/jjb/shell/sync-dir.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# sync-dir.sh - run build step then sync a directory to a remote server
+set -eu -o pipefail
+
+# when not running under Jenkins, use current dir as workspace, a blank project
+# name
+WORKSPACE=${WORKSPACE:-.}
+
+# run the build command
+$BUILD_COMMAND
+
+# sync the files to the target
+rsync -rvzh --delete-after --exclude=.git "$WORKSPACE/$BUILD_OUTPUT_PATH" "$SYNC_TARGET_SERVER:$SYNC_TARGET_PATH"
diff --git a/jjb/shell/tag-check.sh b/jjb/shell/tag-check.sh
new file mode 100755
index 0000000..ea66045
--- /dev/null
+++ b/jjb/shell/tag-check.sh
@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+
+# Copyright 2018-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# tag-check.sh
+# checks that there isn't an existing tag in the git repo that matches what is
+# is in the version file in the commit, to avoid duplicate git tags
+
+set -eu -o pipefail
+
+VERSIONFILE="" # file path to file containing version number
+NEW_VERSION="" # version number found in $VERSIONFILE
+
+SEMVER_STRICT=${SEMVER_STRICT:-0} # require semver versions
+
+releaseversion=0
+fail_validation=0
+
+# when not running under Jenkins, use current dir as workspace
+WORKSPACE=${WORKSPACE:-.}
+
+# find the version string in the repo, read into NEW_VERSION
+# Add additional places NEW_VERSION could be found to this function
+function read_version {
+ if [ -f "VERSION" ]
+ then
+ NEW_VERSION=$(head -n1 "VERSION")
+ VERSIONFILE="VERSION"
+ elif [ -f "package.json" ]
+ then
+ NEW_VERSION=$(python -c 'import json,sys;obj=json.load(sys.stdin); print obj["version"]' < package.json)
+ VERSIONFILE="package.json"
+ else
+ echo "ERROR: No versioning file found!"
+ exit 1
+ fi
+}
+
+# check if the version is already a tag in git
+function is_git_tag_duplicated {
+ for existing_tag in $(git tag)
+ do
+ if [ "$NEW_VERSION" = "$existing_tag" ]
+ then
+ echo "ERROR: Duplicate tag: $existing_tag"
+ exit 2
+ fi
+ done
+}
+
+# check if the version is a released version
+function check_if_releaseversion {
+ if [[ "$NEW_VERSION" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]
+ then
+ echo "Version string '$NEW_VERSION' found in '$VERSIONFILE' is a SemVer released version!"
+ releaseversion=1
+ else
+ if [ "$SEMVER_STRICT" -eq "1" ]
+ then
+ echo "Version string '$NEW_VERSION' in '$VERSIONFILE' is not a SemVer released version, SEMVER_STRICT enabled, failing!"
+ fail_validation=1
+ else
+ echo "Version string '$NEW_VERSION' in '$VERSIONFILE' is not a SemVer released version, skipping."
+ fi
+ fi
+}
+
+# check if Dockerfiles have a released version as their parent
+function dockerfile_parentcheck {
+ while IFS= read -r -d '' dockerfile
+ do
+ echo "Checking dockerfile: '$dockerfile'"
+
+ # split on newlines
+ IFS=$'\n'
+ df_parents=($(grep "^FROM" "$dockerfile"))
+
+ # check all parents in the Dockerfile
+ for df_parent in "${df_parents[@]}"
+ do
+
+ df_pattern="FROM (.*):(.*)"
+ if [[ "$df_parent" =~ $df_pattern ]]
+ then
+
+ p_image="${BASH_REMATCH[1]}"
+ p_version="${BASH_REMATCH[2]}"
+
+ if [[ "${p_version}" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]
+ then
+ echo " OK: Parent '$p_image:$p_version' is a released SemVer version"
+ elif [[ "${p_version}" =~ ^.*([0-9]+)\.([0-9]+).*$ ]]
+ then
+ # handle non-SemVer versions that have a Major.Minor version specifier in the name
+ # 'ubuntu:16.04'
+ # 'postgres:10.3-alpine'
+ # 'openjdk:8-jre-alpine3.8'
+ echo " OK: Parent '$p_image:$p_version' is using a non-SemVer, but sufficient, version"
+ else
+ echo " ERROR: Parent '$p_image:$p_version' is NOT using an specific version"
+ fail_validation=1
+ fi
+
+ elif [[ "$df_parent" =~ ^FROM\ scratch$ ]]
+ then
+ # Handle the parent-less `FROM scratch` case:
+ # https://docs.docker.com/develop/develop-images/baseimages/
+ echo " OK: Using the versionless 'scratch' parent: '$df_parent'"
+ else
+ echo " ERROR: Couldn't find a parent image in $df_parent"
+ fi
+
+ done
+
+ done < <( find "${WORKSPACE}" -name 'Dockerfile*' ! -path "*/vendor/*" -print0 )
+}
+
+echo "Checking git repo with remotes:"
+git remote -v
+
+echo "Branches:"
+git branch -v
+
+echo "Existing git tags:"
+git tag -n
+
+read_version
+check_if_releaseversion
+
+# perform checks if a released version
+if [ "$releaseversion" -eq "1" ]
+then
+ is_git_tag_duplicated
+ dockerfile_parentcheck
+fi
+
+exit $fail_validation
diff --git a/jjb/shell/version-tag.sh b/jjb/shell/version-tag.sh
new file mode 100755
index 0000000..6c7774c
--- /dev/null
+++ b/jjb/shell/version-tag.sh
@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+
+# Copyright 2018-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# version-tag.sh
+# Tags a git commit with the SemVer version discovered within the commit,
+# if the tag doesn't already exist. Ignore non-SemVer commits.
+
+set -eu -o pipefail
+
+VERSIONFILE="" # file path to file containing version number
+NEW_VERSION="" # version number found in $VERSIONFILE
+
+SEMVER_STRICT=${SEMVER_STRICT:-0} # require semver versions
+
+releaseversion=0
+fail_validation=0
+
+# when not running under Jenkins, use current dir as workspace
+WORKSPACE=${WORKSPACE:-.}
+
+# find the version string in the repo, read into NEW_VERSION
+# Add additional places NEW_VERSION could be found to this function
+function read_version {
+ if [ -f "VERSION" ]
+ then
+ NEW_VERSION=$(head -n1 "VERSION")
+ VERSIONFILE="VERSION"
+ elif [ -f "package.json" ]
+ then
+ NEW_VERSION=$(python -c 'import json,sys;obj=json.load(sys.stdin); print obj["version"]' < package.json)
+ VERSIONFILE="package.json"
+ else
+ echo "ERROR: No versioning file found!"
+ exit 1
+ fi
+}
+
+# check if the version is a released version
+function check_if_releaseversion {
+ if [[ "$NEW_VERSION" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]
+ then
+ echo "Version string '$NEW_VERSION' in '$VERSIONFILE' is a SemVer released version!"
+ releaseversion=1
+ else
+ if [ "$SEMVER_STRICT" -eq "1" ]
+ then
+ echo "Version string '$NEW_VERSION' in '$VERSIONFILE' is not a SemVer released version, SEMVER_STRICT enabled, failing!"
+ fail_validation=1
+ else
+ echo "Version string '$NEW_VERSION' in '$VERSIONFILE' is not a SemVer released version, skipping."
+ fi
+ fi
+}
+
+# check if the version is already a tag in git
+function is_git_tag_duplicated {
+ for existing_tag in $(git tag)
+ do
+ if [ "$NEW_VERSION" = "$existing_tag" ]
+ then
+ echo "ERROR: Duplicate tag: $existing_tag"
+ exit 2
+ fi
+ done
+}
+
+# check if Dockerfiles have a released version as their parent
+function dockerfile_parentcheck {
+ while IFS= read -r -d '' dockerfile
+ do
+ echo "Checking dockerfile: '$dockerfile'"
+
+ # split on newlines
+ IFS=$'\n'
+ df_parents=($(grep "^FROM" "$dockerfile"))
+
+ # check all parents in the Dockerfile
+ for df_parent in "${df_parents[@]}"
+ do
+
+ df_pattern="FROM (.*):(.*)"
+ if [[ "$df_parent" =~ $df_pattern ]]
+ then
+
+ p_image="${BASH_REMATCH[1]}"
+ p_version="${BASH_REMATCH[2]}"
+
+ if [[ "${p_version}" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]
+ then
+ echo " OK: Parent '$p_image:$p_version' is a released SemVer version"
+ elif [[ "${p_version}" =~ ^.*([0-9]+)\.([0-9]+).*$ ]]
+ then
+ # handle non-SemVer versions that have a Major.Minor version specifier in the name
+ # 'ubuntu:16.04'
+ # 'postgres:10.3-alpine'
+ # 'openjdk:8-jre-alpine3.8'
+ echo " OK: Parent '$p_image:$p_version' is using a non-SemVer, but sufficient, version"
+ else
+ echo " ERROR: Parent '$p_image:$p_version' is NOT using an specific version"
+ fail_validation=1
+ fi
+
+ elif [[ "$df_parent" =~ ^FROM\ scratch$ ]]
+ then
+ # Handle the parent-less `FROM scratch` case:
+ # https://docs.docker.com/develop/develop-images/baseimages/
+ echo " OK: Using the versionless 'scratch' parent: '$df_parent'"
+ else
+ echo " ERROR: Couldn't find a parent image in $df_parent"
+ fi
+
+ done
+
+ done < <( find "${WORKSPACE}" -name 'Dockerfile*' ! -path "*/vendor/*" -print0 )
+}
+
+
+# create a git tag
+function create_git_tag {
+ echo "Creating git tag: $NEW_VERSION"
+ git checkout "$GERRIT_PATCHSET_REVISION"
+
+ git config --global user.email "do-not-reply@opencord.org"
+ git config --global user.name "Jenkins"
+
+ git tag -a "$NEW_VERSION" -m "Tagged by CORD Jenkins version-tag job: $BUILD_NUMBER, for Gerrit patchset: $GERRIT_CHANGE_NUMBER"
+
+ echo "Tags including new tag:"
+ git tag -n
+
+ git push origin "$NEW_VERSION"
+}
+
+echo "Checking git repo with remotes:"
+git remote -v
+
+echo "Branches:"
+git branch -v
+
+echo "Existing git tags:"
+git tag -n
+
+read_version
+check_if_releaseversion
+
+# perform checks if a released version
+if [ "$releaseversion" -eq "1" ]
+then
+ is_git_tag_duplicated
+ dockerfile_parentcheck
+
+ if [ "$fail_validation" -eq "0" ]
+ then
+ create_git_tag
+ else
+ echo "ERROR: commit merged but failed validation, not tagging!"
+ fi
+fi
+
+exit $fail_validation
diff --git a/jjb/templates/make-test.yaml b/jjb/templates/make-test.yaml
new file mode 100644
index 0000000..2928e64
--- /dev/null
+++ b/jjb/templates/make-test.yaml
@@ -0,0 +1,66 @@
+---
+# Run Makefile targets and optionally collect unit test data
+
+- job-template:
+ id: 'make-test'
+ name: 'make-test{name-extension}_{project}'
+
+ description: |
+ Created by {id} job-template from ci-management/jjb/templates/make-test.yaml<br/>
+ Runs make with the following test targets - '{make-test-targets}'
+
+ triggers:
+ - onf-infra-gerrit-trigger-patchset:
+ gerrit-server-name: '{gerrit-server-name}'
+ project-regexp: '^{project}$'
+ branch-regexp: '{branch-regexp}'
+ dependency-jobs: '{dependency-jobs}'
+ file-include-regexp: '{all-files-regexp}'
+
+ properties:
+ - onf-infra-properties:
+ build-days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+ wrappers:
+ - lf-infra-wrappers:
+ build-timeout: '{build-timeout}'
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+ scm:
+ - onf-infra-gerrit-scm:
+ git-url: '$GIT_URL/$GERRIT_PROJECT'
+ refspec: '$GERRIT_REFSPEC'
+ branch: '$GERRIT_BRANCH'
+ submodule-recursive: 'false'
+ choosing-strategy: gerrit
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
+ basedir: '{project}'
+
+ node: '{build-node}'
+ project-type: freestyle
+ concurrent: true
+
+ builders:
+ - inject:
+ properties-content: |
+ DEST_GOPATH={dest-gopath}
+ MAKE_TEST_TARGETS={make-test-targets}
+ MAKE_TEST_KEEP_GOING={make-test-keep-going}
+ - shell: !include-raw-escape: ../shell/make-test.sh
+
+ publishers:
+ - junit:
+ results: "**/*results.xml,**/*report.xml"
+ allow-empty-results: '{junit-allow-empty-results}'
+ - cobertura:
+ report-file: "**/*coverage.xml"
+ targets:
+ - files:
+ healthy: 80
+ unhealthy: 0
+ failing: 0
+ - method:
+ healthy: 50
+ unhealthy: 0
+ failing: 0
diff --git a/jjb/templates/sync-dir.yaml b/jjb/templates/sync-dir.yaml
new file mode 100644
index 0000000..8faa101
--- /dev/null
+++ b/jjb/templates/sync-dir.yaml
@@ -0,0 +1,65 @@
+---
+# sync built directory to a remote server
+
+- job-template:
+ id: sync-dir
+ name: "sync-dir_{project}"
+ description: |
+ Created by {id} job-template from ci-management/jjb/templates/sync-dir.yaml
+ After merge run a build step then upload files to a directory remote server.
+
+ parameters:
+ - string:
+ name: BUILD_COMMAND
+ default: '{build-command}'
+ description: 'Name of the command to run to generate artifacts'
+
+ - string:
+ name: BUILD_OUTPUT_PATH
+ default: '{build-output-path}'
+ description: 'Path of files where build output is created by build command, relative to code checkout location'
+
+ - string:
+ name: SYNC_TARGET_SERVER
+ default: '{sync-target-server}'
+ description: 'Name of server where built files will be synced'
+
+ - string:
+ name: SYNC_TARGET_PATH
+ default: '{sync-target-path}'
+ description: 'Directory path on target server where the files will be synced'
+
+ triggers:
+ - onf-infra-gerrit-trigger-merge:
+ gerrit-server-name: '{gerrit-server-name}'
+ project-regexp: '^{project}$'
+ branch-regexp: '{branch-regexp}'
+ file-include-regexp: '{all-files-regexp}'
+ dependency-jobs: '{dependency-jobs}'
+
+ properties:
+ - onf-infra-properties:
+ build-days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+ wrappers:
+ - onf-infra-rsync-wrappers:
+ build-timeout: '{build-timeout}'
+ jenkins-ssh-credential: '{gerrit-ssh-credential}'
+
+ scm:
+ - lf-infra-gerrit-scm:
+ git-url: '$GIT_URL/$GERRIT_PROJECT'
+ refspec: '$GERRIT_REFSPEC'
+ branch: '$GERRIT_BRANCH'
+ submodule-recursive: 'false'
+ choosing-strategy: gerrit
+ jenkins-ssh-credential: '{gerrit-ssh-credential}'
+
+ node: '{build-node}'
+ project-type: freestyle
+ concurrent: true
+
+ builders:
+ - shell: !include-raw-escape: ../shell/sync-dir.sh
+
diff --git a/jjb/templates/verify-licensed.yaml b/jjb/templates/verify-licensed.yaml
new file mode 100644
index 0000000..48bf37a
--- /dev/null
+++ b/jjb/templates/verify-licensed.yaml
@@ -0,0 +1,43 @@
+---
+# Verify that there is valid license/copyright on files
+
+- job-template:
+ id: verify-licensed
+ name: 'verify-licensed_{project}'
+ description: |
+ Created by ci-management/jjb/templates/verify-licensed.yaml
+
+ triggers:
+ - onf-infra-gerrit-trigger-patchset:
+ gerrit-server-name: '{gerrit-server-name}'
+ project-regexp: '^{project}$'
+ branch-regexp: '{branch-regexp}'
+ file-include-regexp: '{all-files-regexp}'
+ dependency-jobs: '{dependency-jobs}'
+
+ properties:
+ - onf-infra-properties:
+ build-days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+ wrappers:
+ - lf-infra-wrappers:
+ build-timeout: '{build-timeout}'
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+ scm:
+ - lf-infra-gerrit-scm:
+ git-url: '$GIT_URL/$GERRIT_PROJECT'
+ refspec: '$GERRIT_REFSPEC'
+ branch: '$GERRIT_BRANCH'
+ submodule-recursive: 'false'
+ choosing-strategy: gerrit
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+ node: '{build-node}'
+ project-type: freestyle
+ concurrent: true
+
+ builders:
+ - shell: !include-raw-escape: ../shell/licensecheck.sh
+
diff --git a/jjb/templates/versioning.yaml b/jjb/templates/versioning.yaml
new file mode 100644
index 0000000..9e2f164
--- /dev/null
+++ b/jjb/templates/versioning.yaml
@@ -0,0 +1,111 @@
+---
+# versioning jobs for tagging/releasing software
+
+# Versioning conventions:
+#
+# 1. There is a 1:1 relationship between SemVer _release_ versions described
+# in the commit and the git tag applied to that commit by Jenkins.
+#
+# 2. Non-release versions (ex: 1.0.1-dev3, etc.) can exist in multiple
+# commits, and don't trigger creation of git tags.
+#
+# 3. Git history is public, and therefore shouldn't be rewritten to abandon
+# already merged commits
+#
+# 4. Reverting a commit leaves it in history, so if a broken version is
+# released, the correct action is to make a new fixed version, not try to
+# fix the released version
+#
+# For reference: https://jira.opencord.org/browse/CORD-3117
+
+- job-template:
+ id: tag-check
+ name: "tag-check_{project}"
+ description: |
+ Created by {id} job-template from ci-management/jjb/templates/versioning.yaml
+ Checks for changes to version files, and that they don't duplicate tags
+ already in the git repo.
+
+ triggers:
+ - onf-infra-gerrit-trigger-patchset:
+ gerrit-server-name: '{gerrit-server-name}'
+ project-regexp: '^{project}$'
+ branch-regexp: '{branch-regexp}'
+ file-include-regexp: '{all-files-regexp}'
+ dependency-jobs: '{dependency-jobs}'
+
+ properties:
+ - onf-infra-properties:
+ build-days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+ wrappers:
+ - lf-infra-wrappers:
+ build-timeout: '{build-timeout}'
+ jenkins-ssh-credential: '{gerrit-ssh-credential}'
+
+ scm:
+ - lf-infra-gerrit-scm:
+ git-url: '$GIT_URL/$GERRIT_PROJECT'
+ refspec: '$GERRIT_REFSPEC'
+ branch: '$GERRIT_BRANCH'
+ submodule-recursive: 'false'
+ choosing-strategy: gerrit
+ jenkins-ssh-credential: '{gerrit-ssh-credential}'
+
+ node: '{build-node}'
+ project-type: freestyle
+ concurrent: true
+
+ builders:
+ - inject:
+ properties-content:
+ SEMVER_STRICT={semver-strict}
+ - shell: !include-raw-escape: ../shell/tag-check.sh
+
+
+- job-template:
+ id: version-tag
+ name: "version-tag_{project}"
+ description: |
+ Created by {id} job-template from ci-management/jjb/templates/versioning.yaml
+ When a patch is merged, check if it contains a SemVer released version
+ file and if so tags the commit in git with that same version.
+
+ triggers:
+ - onf-infra-gerrit-trigger-merge:
+ gerrit-server-name: '{gerrit-server-name}'
+ project-regexp: '^{project}$'
+ branch-regexp: '{branch-regexp}'
+ file-include-regexp: '{all-files-regexp}'
+ dependency-jobs: '{dependency-jobs}'
+
+ properties:
+ - onf-infra-properties:
+ build-days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+ wrappers:
+ - lf-infra-wrappers:
+ build-timeout: '{build-timeout}'
+ jenkins-ssh-credential: '{gerrit-ssh-credential}'
+
+ scm:
+ - lf-infra-gerrit-scm:
+ git-url: '$GIT_URL/$GERRIT_PROJECT'
+ refspec: '$GERRIT_REFSPEC'
+ branch: '$GERRIT_BRANCH'
+ submodule-recursive: 'false'
+ choosing-strategy: gerrit
+ jenkins-ssh-credential: '{gerrit-ssh-credential}'
+
+ node: '{build-node}'
+ project-type: freestyle
+ concurrent: true
+
+ builders:
+ - inject:
+ properties-content:
+ SEMVER_STRICT={semver-strict}
+ - shell: !include-raw-escape: ../shell/version-tag.sh
+
diff --git a/packer/provision/basebuild.sh b/packer/provision/basebuild.sh
index acd92a8..b41e9f9 100644
--- a/packer/provision/basebuild.sh
+++ b/packer/provision/basebuild.sh
@@ -30,26 +30,29 @@
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
-
+
apt-get update
apt-get install -y \
bzip2 \
+ chromium-browser \
curl \
+ docker-ce \
git \
less \
- oracle-java8-installer \
- oracle-java8-set-default \
- python \
- ssh \
- zip \
maven \
nodejs \
nodejs-legacy \
npm \
+ oracle-java8-installer \
+ oracle-java8-set-default \
+ python \
python-pip \
- docker-ce \
- chromium-browser \
+ python-virtualenv \
+ rsync \
+ ssh \
+ zip
# end of apt-get install list
+
npm install -g bower
npm install karma --save-dev