From 868ab36421fe3136a83beddb1ef98b4e489fd9e9 Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Wed, 31 Jul 2019 15:32:03 -0400 Subject: [PATCH] Define Docker-CI-PC merge job template Add new JJB template that builds in docker, exports files then pushes packages to PackageCloud.io. Define default values for account, repo and distributions. Reuse packagecloud-push.sh script created by suresh channamallu. Add new merge+push job to projects that build DEB/RPM packages. Signed-off-by: Lott, Christopher (cl778h) Change-Id: I8365ce8b37427692f8c6c436e87bd767bfa996c7 --- jjb/com-log/com-log.yaml | 5 +- jjb/defaults.yaml | 6 + jjb/oran-jjb/oran-docker-ci-jobs.yaml | 187 ++++++++++++++++++++++++++++ jjb/{ => oran-jjb}/shell/export-packages.sh | 3 +- jjb/oran-jjb/shell/packagecloud-push.sh | 41 ++++++ jjb/ric-plt-lib-rmr/ric-plt-lib-rmr.yaml | 5 +- 6 files changed, 238 insertions(+), 9 deletions(-) create mode 100644 jjb/oran-jjb/oran-docker-ci-jobs.yaml rename jjb/{ => oran-jjb}/shell/export-packages.sh (92%) create mode 100755 jjb/oran-jjb/shell/packagecloud-push.sh diff --git a/jjb/com-log/com-log.yaml b/jjb/com-log/com-log.yaml index 5465a473..ffd23b76 100644 --- a/jjb/com-log/com-log.yaml +++ b/jjb/com-log/com-log.yaml @@ -34,12 +34,9 @@ container-tag-method: latest # maven settings file has docker credentials mvn-settings: com-log-settings - # copy packages to Jenkins filesystem - post_docker_build_script: !include-raw-escape: - - ../shell/export-packages.sh jobs: - # TODO: define a merge template to push package, not image - gerrit-docker-verify + - oran-gerrit-docker-ci-pc-merge <<: *log_docker_common - project: diff --git a/jjb/defaults.yaml b/jjb/defaults.yaml index 01e88396..95041f94 100644 --- a/jjb/defaults.yaml +++ b/jjb/defaults.yaml @@ -35,3 +35,9 @@ # Javadoc javadoc-path: org.o-ran-sc.$PROJECT/$STREAM + + # Config for PackageCloud.IO archive of DEB/RPM packages + packagecloud-account: oran + packagecloud-repo: staging + debian-distribution-versions: debian/stretch + rpm-distribution-versions: el/5 diff --git a/jjb/oran-jjb/oran-docker-ci-jobs.yaml b/jjb/oran-jjb/oran-docker-ci-jobs.yaml new file mode 100644 index 00000000..68a89083 --- /dev/null +++ b/jjb/oran-jjb/oran-docker-ci-jobs.yaml @@ -0,0 +1,187 @@ +--- +# Copyright (c) 2019 AT&T Intellectual Property. +# Copyright (c) 2019 Nokia. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Macros and templates for using Docker as the CI system. +# The merge template calls builders to build in docker, +# export files, then push to PackageCloud. +# The docker image is discarded, not pushed to a registry. +# Uses builders defined in, and mirrors the structure of, +# global-jjb resource jjb/lf-docker-jobs.yaml + +- builder: + name: oran-packagecloud-file-provider + builders: + - config-file-provider: + files: + - file-id: ".packagecloud" + target: "$HOME/.packagecloud" + - config-file-provider: + files: + - file-id: "packagecloud_api" + target: "$HOME/packagecloud_api" + +# This shell script launches the container, which copies +# build products to a mounted volume. Script needs vars +# CONTAINER_PUSH_REGISTRY, DOCKER_NAME and DOCKER_IMAGE_TAG +# as set by lf-docker-get-container-tag and lf-docker-build. +- builder: + name: oran-docker-build-export + builders: + - shell: !include-raw-escape: + - shell/export-packages.sh + +- builder: + # {packagecloud-account} is PackageCloud account ID; example: oran + # {packagecloud-repo} is PackageCloud repository; example: master, staging + # {debian-distribution-versions} is list of DEB package distro/version strings + # separated by space; example: ubuntu/bionic debian/stretch + # {rpm-distribution-versions} is list of RPM package distro/version strings + # separated by space; example: el/4 el/5 + name: oran-packagecloud-push + builders: + - inject: + properties-content: | + PACKAGECLOUD_ACCOUNT={packagecloud-account} + PACKAGECLOUD_REPO={packagecloud-repo} + DEBIAN_DISTRIBUTION_VERSIONS={debian-distribution-versions} + RPM_DISTRIBUTION_VERSIONS={rpm-distribution-versions} + - shell: !include-raw-escape: + - shell/packagecloud-push.sh + +# Different name, same content as lf_docker_common +- oran_docker_common: &oran_docker_common + name: oran-docker-ci-common + + project-type: freestyle + node: "{build-node}" + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 7 + build-timeout: 60 + submodule-disable: false + submodule-recursive: true + submodule-timeout: 10 + pre_docker_build_script: "# pre docker build script goes here" + post_docker_build_script: "# post docker build script goes here" + disable-job: "false" + docker-root: "$WORKSPACE" + docker-build-args: "" + git-url: "$GIT_URL/$PROJECT" + # TODO: how to interpolate value of {docker-root} parameter? + container-tag-yaml-dir: "" + + ##################### + # Job Configuration # + ##################### + + disabled: "{disable-job}" + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + + parameters: + - lf-infra-parameters: + branch: "{branch}" + project: "{project}" + refspec: "refs/heads/{branch}" + stream: "{stream}" + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + publishers: + - lf-infra-publish + +- oran_docker_ci_pc_merge_common: &oran_docker_ci_pc_merge_common + name: oran-docker-ci-pc-merge-common + cron: "" + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-disable: "{submodule-disable}" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + choosing-strategy: gerrit + + gerrit_merge_triggers: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: remerge$ + + gerrit_trigger_file_paths: + - compare-type: REG_EXP + pattern: ".*" + + builders: + - lf-infra-pre-build + - config-file-provider: + files: + - file-id: ".packagecloud" + target: "$HOME/.packagecloud" + - lf-infra-docker-login: + global-settings-file: "global-settings" + settings-file: "{mvn-settings}" + - shell: "{pre_docker_build_script}" + - lf-docker-get-container-tag: + container-tag-method: "{container-tag-method}" + container-tag-yaml-dir: "{container-tag-yaml-dir}" + docker-root: "{docker-root}" + - lf-docker-build: + docker-build-args: "{docker-build-args}" + docker-name: "{docker-name}" + docker-root: "{docker-root}" + container-public-registry: "{container-public-registry}" + container-push-registry: "{container-push-registry}" + - shell: "{post_docker_build_script}" + - oran-docker-build-export + - oran-packagecloud-push: + packagecloud-account: "{packagecloud-account}" + packagecloud-repo: "{packagecloud-repo}" + debian-distribution-versions: "{debian-distribution-versions}" + rpm-distribution-versions: "{rpm-distribution-versions}" + - lf-provide-maven-settings-cleanup + +- job-template: + name: "{project-name}-docker-ci-pc-merge-{stream}" + id: oran-gerrit-docker-ci-pc-merge + + <<: *oran_docker_common + # yamllint disable-line rule:key-duplicates + <<: *oran_docker_ci_pc_merge_common + + triggers: + - timed: "{obj:cron}" + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_merge_triggers}" + projects: + - project-compare-type: ANT + project-pattern: "{project}" + branches: + - branch-compare-type: ANT + branch-pattern: "**/{branch}" + file-paths: "{obj:gerrit_trigger_file_paths}" diff --git a/jjb/shell/export-packages.sh b/jjb/oran-jjb/shell/export-packages.sh similarity index 92% rename from jjb/shell/export-packages.sh rename to jjb/oran-jjb/shell/export-packages.sh index 68957e9a..e6bf0768 100755 --- a/jjb/shell/export-packages.sh +++ b/jjb/oran-jjb/shell/export-packages.sh @@ -22,7 +22,8 @@ echo "--> export-packages.sh" # Launches the docker image, which has an entrypoint script # that copies artifacts created by the builder to a directory. # Environment variables are injected in previous Jenkins steps. -HOST=/tmp/export +# The push script searches the workspace, so do not use /tmp etc. +HOST=$WORKSPACE/export-packages-$$ GUEST=/export mkdir -p "$HOST" docker run -v "$HOST":"$GUEST" "$CONTAINER_PUSH_REGISTRY"/"$DOCKER_NAME":"$DOCKER_IMAGE_TAG" "$GUEST" diff --git a/jjb/oran-jjb/shell/packagecloud-push.sh b/jjb/oran-jjb/shell/packagecloud-push.sh new file mode 100755 index 00000000..a48d39cf --- /dev/null +++ b/jjb/oran-jjb/shell/packagecloud-push.sh @@ -0,0 +1,41 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> packagecloud-push.sh" +set -eu -o pipefail + +if [ ! -f ~/.packagecloud ]; then + echo "INFO: .packagecloud file not found" + exit 0 +fi + + # For DEB +vers=("$DEBIAN_DISTRIBUTION_VERSIONS") +echo "Debian distribution versions:" "${vers[@]}" +debs=$(find . -type f -iname '*.deb') +# modern bash syntax is helpful +for (( i = 0; i < ${#vers[@]}; i++ )); do + for deb in $debs; do + echo "Pushing $deb $PACKAGECLOUD_ACCOUNT/$PACKAGECLOUD_REPO/${vers[i]}" + package_cloud push "$PACKAGECLOUD_ACCOUNT"/"$PACKAGECLOUD_REPO"/"${vers[i]}" "$deb" + done +done + +# For RPM +vers=("$RPM_DISTRIBUTION_VERSIONS") +echo "RPM distribution versions:" "${vers[@]}" +rpms=$(find . -type f -iregex '.*/.*\.\(s\)?rpm') +# modern bash syntax is helpful +for (( i = 0; i < ${#vers[@]}; i++ )); do + for rpm in $rpms; do + echo "Pushing $rpm $PACKAGECLOUD_ACCOUNT/$PACKAGECLOUD_REPO/${vers[i]}" + package_cloud push "$PACKAGECLOUD_ACCOUNT"/"$PACKAGECLOUD_REPO"/"${vers[i]}" "$rpm" + done +done diff --git a/jjb/ric-plt-lib-rmr/ric-plt-lib-rmr.yaml b/jjb/ric-plt-lib-rmr/ric-plt-lib-rmr.yaml index a27fb76c..aba81aad 100644 --- a/jjb/ric-plt-lib-rmr/ric-plt-lib-rmr.yaml +++ b/jjb/ric-plt-lib-rmr/ric-plt-lib-rmr.yaml @@ -34,12 +34,9 @@ container-tag-method: latest # maven settings file has docker credentials mvn-settings: ric-plt-lib-rmr-settings - # copy packages to Jenkins filesystem - post_docker_build_script: !include-raw-escape: - - ../shell/export-packages.sh jobs: - # TODO: define a merge template to push package, not image - gerrit-docker-verify + - oran-gerrit-docker-ci-pc-merge <<: *rmr_docker_common - project: -- 2.16.6