Define Docker-CI-PC merge job template 20/620/5
authorLott, Christopher (cl778h) <cl778h@att.com>
Wed, 31 Jul 2019 19:32:03 +0000 (15:32 -0400)
committerLott, Christopher (cl778h) <cl778h@att.com>
Fri, 2 Aug 2019 20:41:09 +0000 (16:41 -0400)
Add new JJB template that builds in docker, exports files
then pushes packages to PackageCloud.io.
Define default values for account, repo and distributions.
Reuse packagecloud-push.sh script created by suresh channamallu.
Add new merge+push job to projects that build DEB/RPM packages.

Signed-off-by: Lott, Christopher (cl778h) <cl778h@att.com>
Change-Id: I8365ce8b37427692f8c6c436e87bd767bfa996c7

jjb/com-log/com-log.yaml
jjb/defaults.yaml
jjb/oran-jjb/oran-docker-ci-jobs.yaml [new file with mode: 0644]
jjb/oran-jjb/shell/export-packages.sh [moved from jjb/shell/export-packages.sh with 92% similarity]
jjb/oran-jjb/shell/packagecloud-push.sh [new file with mode: 0755]
jjb/ric-plt-lib-rmr/ric-plt-lib-rmr.yaml

index 5465a47..ffd23b7 100644 (file)
     container-tag-method: latest
     # maven settings file has docker credentials
     mvn-settings: com-log-settings
-    # copy packages to Jenkins filesystem
-    post_docker_build_script: !include-raw-escape:
-      - ../shell/export-packages.sh
     jobs:
-      # TODO: define a merge template to push package, not image
       - gerrit-docker-verify
+      - oran-gerrit-docker-ci-pc-merge
     <<: *log_docker_common
 
 - project:
index 01e8839..95041f9 100644 (file)
@@ -35,3 +35,9 @@
 
     # Javadoc
     javadoc-path: org.o-ran-sc.$PROJECT/$STREAM
+
+    # Config for PackageCloud.IO archive of DEB/RPM packages
+    packagecloud-account: oran
+    packagecloud-repo: staging
+    debian-distribution-versions: debian/stretch
+    rpm-distribution-versions: el/5
diff --git a/jjb/oran-jjb/oran-docker-ci-jobs.yaml b/jjb/oran-jjb/oran-docker-ci-jobs.yaml
new file mode 100644 (file)
index 0000000..68a8908
--- /dev/null
@@ -0,0 +1,187 @@
+---
+#   Copyright (c) 2019 AT&T Intellectual Property.
+#   Copyright (c) 2019 Nokia.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# Macros and templates for using Docker as the CI system.
+# The merge template calls builders to build in docker,
+# export files, then push to PackageCloud.
+# The docker image is discarded, not pushed to a registry.
+# Uses builders defined in, and mirrors the structure of,
+# global-jjb resource jjb/lf-docker-jobs.yaml
+
+- builder:
+    name: oran-packagecloud-file-provider
+    builders:
+      - config-file-provider:
+          files:
+            - file-id: ".packagecloud"
+              target: "$HOME/.packagecloud"
+      - config-file-provider:
+          files:
+            - file-id: "packagecloud_api"
+              target: "$HOME/packagecloud_api"
+
+# This shell script launches the container, which copies
+# build products to a mounted volume.  Script needs vars
+# CONTAINER_PUSH_REGISTRY, DOCKER_NAME and DOCKER_IMAGE_TAG
+# as set by lf-docker-get-container-tag and lf-docker-build.
+- builder:
+    name: oran-docker-build-export
+    builders:
+      - shell: !include-raw-escape:
+          - shell/export-packages.sh
+
+- builder:
+    # {packagecloud-account} is PackageCloud account ID; example: oran
+    # {packagecloud-repo} is PackageCloud repository; example: master, staging
+    # {debian-distribution-versions} is list of DEB package distro/version strings
+    #    separated by space; example: ubuntu/bionic debian/stretch
+    # {rpm-distribution-versions} is list of RPM package distro/version strings
+    #    separated by space; example: el/4 el/5
+    name: oran-packagecloud-push
+    builders:
+      - inject:
+          properties-content: |
+            PACKAGECLOUD_ACCOUNT={packagecloud-account}
+            PACKAGECLOUD_REPO={packagecloud-repo}
+            DEBIAN_DISTRIBUTION_VERSIONS={debian-distribution-versions}
+            RPM_DISTRIBUTION_VERSIONS={rpm-distribution-versions}
+      - shell: !include-raw-escape:
+          - shell/packagecloud-push.sh
+
+# Different name, same content as lf_docker_common
+- oran_docker_common: &oran_docker_common
+    name: oran-docker-ci-common
+
+    project-type: freestyle
+    node: "{build-node}"
+
+    ######################
+    # Default parameters #
+    ######################
+
+    branch: master
+    build-days-to-keep: 7
+    build-timeout: 60
+    submodule-disable: false
+    submodule-recursive: true
+    submodule-timeout: 10
+    pre_docker_build_script: "# pre docker build script goes here"
+    post_docker_build_script: "# post docker build script goes here"
+    disable-job: "false"
+    docker-root: "$WORKSPACE"
+    docker-build-args: ""
+    git-url: "$GIT_URL/$PROJECT"
+    # TODO: how to interpolate value of {docker-root} parameter?
+    container-tag-yaml-dir: ""
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    disabled: "{disable-job}"
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: "{build-days-to-keep}"
+
+    parameters:
+      - lf-infra-parameters:
+          branch: "{branch}"
+          project: "{project}"
+          refspec: "refs/heads/{branch}"
+          stream: "{stream}"
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: "{build-timeout}"
+          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+
+    publishers:
+      - lf-infra-publish
+
+- oran_docker_ci_pc_merge_common: &oran_docker_ci_pc_merge_common
+    name: oran-docker-ci-pc-merge-common
+    cron: ""
+
+    scm:
+      - lf-infra-gerrit-scm:
+          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+          git-url: "{git-url}"
+          refspec: "$GERRIT_REFSPEC"
+          branch: "$GERRIT_BRANCH"
+          submodule-disable: "{submodule-disable}"
+          submodule-recursive: "{submodule-recursive}"
+          submodule-timeout: "{submodule-timeout}"
+          choosing-strategy: gerrit
+
+    gerrit_merge_triggers:
+      - change-merged-event
+      - comment-added-contains-event:
+          comment-contains-value: remerge$
+
+    gerrit_trigger_file_paths:
+      - compare-type: REG_EXP
+        pattern: ".*"
+
+    builders:
+      - lf-infra-pre-build
+      - config-file-provider:
+          files:
+            - file-id: ".packagecloud"
+              target: "$HOME/.packagecloud"
+      - lf-infra-docker-login:
+          global-settings-file: "global-settings"
+          settings-file: "{mvn-settings}"
+      - shell: "{pre_docker_build_script}"
+      - lf-docker-get-container-tag:
+          container-tag-method: "{container-tag-method}"
+          container-tag-yaml-dir: "{container-tag-yaml-dir}"
+          docker-root: "{docker-root}"
+      - lf-docker-build:
+          docker-build-args: "{docker-build-args}"
+          docker-name: "{docker-name}"
+          docker-root: "{docker-root}"
+          container-public-registry: "{container-public-registry}"
+          container-push-registry: "{container-push-registry}"
+      - shell: "{post_docker_build_script}"
+      - oran-docker-build-export
+      - oran-packagecloud-push:
+          packagecloud-account: "{packagecloud-account}"
+          packagecloud-repo: "{packagecloud-repo}"
+          debian-distribution-versions: "{debian-distribution-versions}"
+          rpm-distribution-versions: "{rpm-distribution-versions}"
+      - lf-provide-maven-settings-cleanup
+
+- job-template:
+    name: "{project-name}-docker-ci-pc-merge-{stream}"
+    id: oran-gerrit-docker-ci-pc-merge
+
+    <<: *oran_docker_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *oran_docker_ci_pc_merge_common
+
+    triggers:
+      - timed: "{obj:cron}"
+      - gerrit:
+          server-name: "{gerrit-server-name}"
+          trigger-on: "{obj:gerrit_merge_triggers}"
+          projects:
+            - project-compare-type: ANT
+              project-pattern: "{project}"
+              branches:
+                - branch-compare-type: ANT
+                  branch-pattern: "**/{branch}"
+              file-paths: "{obj:gerrit_trigger_file_paths}"
similarity index 92%
rename from jjb/shell/export-packages.sh
rename to jjb/oran-jjb/shell/export-packages.sh
index 68957e9..e6bf076 100755 (executable)
@@ -22,7 +22,8 @@ echo "--> export-packages.sh"
 # Launches the docker image, which has an entrypoint script
 # that copies artifacts created by the builder to a directory.
 # Environment variables are injected in previous Jenkins steps.
-HOST=/tmp/export
+# The push script searches the workspace, so do not use /tmp etc.
+HOST=$WORKSPACE/export-packages-$$
 GUEST=/export
 mkdir -p "$HOST"
 docker run -v "$HOST":"$GUEST" "$CONTAINER_PUSH_REGISTRY"/"$DOCKER_NAME":"$DOCKER_IMAGE_TAG" "$GUEST"
diff --git a/jjb/oran-jjb/shell/packagecloud-push.sh b/jjb/oran-jjb/shell/packagecloud-push.sh
new file mode 100755 (executable)
index 0000000..a48d39c
--- /dev/null
@@ -0,0 +1,41 @@
+#!/bin/bash -l
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+echo "---> packagecloud-push.sh"
+set -eu -o pipefail
+
+if [ ! -f ~/.packagecloud ]; then
+    echo "INFO: .packagecloud file not found"
+    exit 0
+fi
+
+ # For DEB
+vers=("$DEBIAN_DISTRIBUTION_VERSIONS")
+echo "Debian distribution versions:" "${vers[@]}"
+debs=$(find . -type f -iname '*.deb')
+# modern bash syntax is helpful
+for (( i = 0; i < ${#vers[@]}; i++ )); do
+    for deb in $debs; do
+       echo "Pushing $deb $PACKAGECLOUD_ACCOUNT/$PACKAGECLOUD_REPO/${vers[i]}"
+       package_cloud push "$PACKAGECLOUD_ACCOUNT"/"$PACKAGECLOUD_REPO"/"${vers[i]}" "$deb"
+    done
+done
+
+# For RPM
+vers=("$RPM_DISTRIBUTION_VERSIONS")
+echo "RPM distribution versions:" "${vers[@]}"
+rpms=$(find . -type f -iregex '.*/.*\.\(s\)?rpm')
+# modern bash syntax is helpful
+for (( i = 0; i < ${#vers[@]}; i++ )); do
+    for rpm in $rpms; do
+       echo "Pushing $rpm $PACKAGECLOUD_ACCOUNT/$PACKAGECLOUD_REPO/${vers[i]}"
+       package_cloud push "$PACKAGECLOUD_ACCOUNT"/"$PACKAGECLOUD_REPO"/"${vers[i]}" "$rpm"
+    done
+done
index a27fb76..aba81aa 100644 (file)
     container-tag-method: latest
     # maven settings file has docker credentials
     mvn-settings: ric-plt-lib-rmr-settings
-    # copy packages to Jenkins filesystem
-    post_docker_build_script: !include-raw-escape:
-      - ../shell/export-packages.sh
     jobs:
-      # TODO: define a merge template to push package, not image
       - gerrit-docker-verify
+      - oran-gerrit-docker-ci-pc-merge
     <<: *rmr_docker_common
 
 - project: