From d4f18ba140f06b863ac51697d1a0d420bcd3ae6a Mon Sep 17 00:00:00 2001 From: "E. Scott Daniels" Date: Mon, 17 Jun 2019 15:08:17 -0400 Subject: [PATCH] enhance(ci): Add multiple package build The CI docker file now drives a CI build script which will execute the unit and application based tests, and if successful will build run-time and development .deb and .rpm packages. Packages are left in /tmp by default and are indexed by build_packages.yml which is also left in /tmp. Change-Id: Ieaa97e1cab1d9cbb721e19d667192e336becec08 Signed-off-by: E. Scott Daniels --- BUILD | 37 ++++++++++---- CMakeLists.txt | 19 ++++--- ci/Dockerfile | 14 +++--- ci/ci_build.ksh | 145 +++++++++++++++++++++++++++++++++++++++++++++++++++++ doc/CMakeLists.txt | 25 +++++---- 5 files changed, 204 insertions(+), 36 deletions(-) create mode 100644 ci/ci_build.ksh diff --git a/BUILD b/BUILD index 3bc677e..37a5045 100644 --- a/BUILD +++ b/BUILD @@ -40,21 +40,38 @@ To build RMr, the usual CMake steps are followed: cmake .. [options] make package + This will create a .deb (provided the system supports this) in the build directory. It's that simple. -Continuous integration build +Continuous Integration Build Use the Dockerfile in the ci/ subdirectory. This installs all -the required tools and creates an image in the local registry. +the required tools, then builds RMr and executes the unit and +programm tests. If tests pass, then an image is created in the +local registry with both run-time and development packages. To support the distribution of package(s) created during the -build by the CI process, the fully qualified path of each generated -package will be placed into a well known YAML file: -/tmp/build_output.yml. This file is created during CMake -configuration and lists the package name(s) for packages which -can be generated given the current environment. Currently -Debian (.deb), and RPM packages are supported (the Ubuntu -alien package must be installed in order to generate RPMs). +build by the CI process, a YAML file is left in the /tmp +directory (build_packages.yml) which contains a list of the +packages available from the image. Currently, both .deb and +.rpm packages are generated. + +The following is a sample YAML file generated during this process: + + # package types which might be listed below + --- + pkg_types: + - deb + - rpm + packages: + - development: + deb: /tmp/rmr-dev_1.0.34_x86_64.deb + rpm: /tmp/rmr-dev-1.0.34-x86_64.rpm + - runtime: + deb: /tmp/rmr_1.0.34_x86_64.deb + rpm: /tmp/rmr-1.0.34-x86_64.rpm + ... + Alternatives @@ -100,7 +117,7 @@ a different spot (e.g. in $HOME/usr): Libraries RMr supports both NNG and Nanomsg as underlying transport. They -are separate beasts, and while an NNG based programme can +are separate beasts, and while an NNG based program can communicate with a Nanomsg based programme, their APIs are NOT compatible. For this reason, and others, RMr generates two libraries and requires that the underlying transport be selected diff --git a/CMakeLists.txt b/CMakeLists.txt index c9ca0ed..7c6e1cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,7 +79,7 @@ add_definitions( -DGIT_ID=${git_id} -DMAJOR_VER=${major_version} -DMINOR_VER=${minor_version} - -DPATCH_VER=${patch_level}${spoiled_str} + -DPATCH_VER=${patch_level} ) # ---------------- suss out pkg gen tools so we don't fail generating packages that the system cannot support -------------- @@ -87,9 +87,14 @@ add_definitions( if( DEV_PKG ) set( dev_tag "-dev" ) + set( pkg_name "rmr-dev" ) +else() + set( pkg_name "rmr" ) endif() -set( pkg_label "${CMAKE_PROJECT_NAME}-${major_version}.${minor_version}.${patch_level}.${CMAKE_SYSTEM_PROCESSOR}${dev_tag}" ) -message( "+++ building ${pkg_label}${spoiled_str}" ) +set( rpm_pkg_label "${CMAKE_PROJECT_NAME}${dev_tag}${spoiled_str}-${major_version}.${minor_version}.${patch_level}-${CMAKE_SYSTEM_PROCESSOR}" ) +set( deb_pkg_label "${CMAKE_PROJECT_NAME}${dev_tag}${spoiled_str}_${major_version}.${minor_version}.${patch_level}_${CMAKE_SYSTEM_PROCESSOR}" ) +message( "+++ building ${deb_pkg_label}" ) +message( "+++ building ${rpm_pkg_label}" ) set( out_yml /tmp/build_output.yml ) # we will record package names (we record only untainted names) find_program( rpm NAMES rpmbuild ) # rpm package gen requires this to be installed @@ -108,12 +113,12 @@ execute_process( ) execute_process( - COMMAND bash -c " echo deb: ${CMAKE_CURRENT_BINARY_DIR}/${pkg_label}.deb >>${out_yml}" + COMMAND bash -c " echo deb: ${CMAKE_CURRENT_BINARY_DIR}/${deb_pkg_label}.deb >>${out_yml}" ) if( gen_rpm ) execute_process( - COMMAND bash -c " echo rpm: ${CMAKE_CURRENT_BINARY_DIR}/${pkg_label}.rpm >>${out_yml}" + COMMAND bash -c " echo rpm: ${CMAKE_CURRENT_BINARY_DIR}/${rpm_pkg_label}.rpm >>${out_yml}" ) endif() @@ -314,6 +319,7 @@ unset( PACK_EXTERNALS CACHE ) IF( EXISTS "${CMAKE_ROOT}/Modules/CPack.cmake" ) include( InstallRequiredSystemLibraries ) + set( CPACK_PACKAGE_NAME ${pkg_name} ) set( CPACK_set_DESTDIR "on" ) set( CPACK_PACKAGING_INSTALL_PREFIX "${install_root}" ) set( CPACK_GENERATOR "${pkg_list}" ) @@ -325,7 +331,8 @@ IF( EXISTS "${CMAKE_ROOT}/Modules/CPack.cmake" ) set( CPACK_PACKAGE_VERSION_MAJOR "${major_version}" ) set( CPACK_PACKAGE_VERSION_MINOR "${minor_version}" ) set( CPACK_PACKAGE_VERSION_PATCH "${patch_level}" ) - set( CPACK_PACKAGE_FILE_NAME "${pkg_label}${spoiled_str}" ) + set( CPACK_DEBIAN_FILE_NAME "${deb_pkg_label}.deb" ) + set( CPACK_RPM_FILE_NAME "${rpm_pkg_label}.rpm" ) # we build and ship the libraries, so there is NO dependency diff --git a/ci/Dockerfile b/ci/Dockerfile index cd81fa3..579adfa 100755 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -14,15 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -# CI to verify the RMR library +# CI to verify the RMR library and build run-time and dev packages # Inherits C toolchain from buildpack-deps:stretch -# Adds cmake for RMR +# Adds cmake ksh and alien for RMR FROM buildpack-deps:stretch RUN apt-get update && apt-get -q -y install cmake ksh alien ADD . /tmp -# tests require directory named ".build" and dev_pkg to place header files in .build -RUN cd /tmp && mkdir .build && cd .build && cmake .. -DDEV_PKG=1 && make install && make package -# tests require a Linux host, fail in a Linux VM on mac/win -RUN cd /tmp/test && ksh unit_test.ksh -v -RUN cd /tmp/test/app_test && ksh run_all.ksh +WORKDIR /tmp + +# build RMr, run unit tests, and generate packages +RUN ksh ci/ci_build.ksh + diff --git a/ci/ci_build.ksh b/ci/ci_build.ksh new file mode 100644 index 0000000..31c821c --- /dev/null +++ b/ci/ci_build.ksh @@ -0,0 +1,145 @@ +#!/usr/bin/env ksh +# :vim ts=4 sw=4 noet: +#================================================================================== +# Copyright (c) 2019 Nokia +# Copyright (c) 2018-2019 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#================================================================================== +# + +# Mnemonic: ci_build.ksh +# Abstract: Script builds RMr, exececutes the unit and application based +# tests, then generates packages. The packages are left in /tmp +# unless -t target-dir is given on the command line. A YAML file +# is created which can be used to find all of the packages which +# were deposited in the target directory. In an environment which +# is also capable of generating RPM ppackages, there should be +# four packages: a run-time and development package for both +# debian (.deb) and rh (rpm) based systems. +# +# The intent of this script is for it to be executed as a part +# of a docker image build process such that the resulting image +# contains the RMr packages, and a YAML file which provides the +# paths to the package(s) in the image. Docker tools can then +# be used to extract the packages and push them to some external +# repository. +# +# Assumptions: +# We assume that this scirpt is executed at the 'root' of the +# RMr repo (i.e. the directory which has a subdirectory ci). +# +# Date: 14 June 2019 +# -------------------------------------------------------------------------------- + +# stash a set of packages for a particular flavour ($1) +# +function stash_pkgs { + echo " - $1:" >>$yaml_file # add package flavour (dev, runtime, etc) + + for pkg in deb rpm + do + ls .build/*.$pkg 2>/dev/null | while read f + do + cp $f $target_dir/${f##*/} + echo " $pkg: $target_dir/${f##*/}" >>$yaml_file + done + + done +} + +# --------------------------------------------------------------------------- + +target_dir="/tmp" +verbose=0 + +while [[ $1 == -* ]] +do + case $1 in + -t) target_dir=$2; shift;; + -v) verbose=1;; + + *) echo "$1 is not recognised" + echo "" + echo "usage: $0 [-t target-dir]" + exit 1 + ;; + esac + + shift +done + +if [[ ! -d $target_dir ]] +then + echo "[FAIL] cannot find directory: $target_dir" + exit 1 +fi + +if [[ ! -d ./ci ]] # verify we are in the root of the RMr repo filesystem, abort if not +then + echo "[FAIL] current working directory does not seem right; should be RMr repo root: $PWD" + exit 1 +fi + +set -e # fail unconditionally on first issue +yaml_file=$target_dir/build_packages.yml +rm -f $yaml_file + +mkdir -p .build +( + cd .build + rm -f *.deb *.rpm # these shouldn't be there, but take no chances + cmake .. -DBUILD_DOC=1 -DDEV_PKG=1 # set up dev config for unit test (requires dev stuff) + make package +) +( + cd test # execute tests + ksh unit_test.ksh # unit tests first + cd app_test + ksh run_all.ksh # application based tests if units pass +) + +# initialise the yaml file +cat <<-endKat >$yaml_file +--- +# package types which might be listed below +pkg_types: + - deb + - rpm + +packages: +endKat + +stash_pkgs development # testing good, stash dev packages built above + +( + cd .build # now build the run-time packages + rm -f *.deb *.rpm # reconfig should delete these, but take no chances + cmake .. # configure run-time build + make package +) +stash_pkgs runtime # move packages to target and record in yaml file + +echo "..." >>$yaml_file # terminate yaml file + +set +e +if (( verbose )) +then + echo "generated yaml file:" + cat $yaml_file + echo "" + echo "------" + ls -al $target_dir/*.deb $target_dir/*.rpm +fi + +exit 0 diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 7675bdc..a4be121 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,7 +1,7 @@ # #================================================================================== -# Copyright (c) 2019 Nokia +# Copyright (c) 2019 Nokia # Copyright (c) 2018-2019 AT&T Intellectual Property. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -39,7 +39,7 @@ if( BUILD_DOC ) if( NOT EXISTS ${tfm} ) # not yet built here, pull and build # pull and build {X}fm tools needed to generate manpages - execute_process( + execute_process( COMMAND "bash" "-c" "git clone https://github.com/ScottDaniels/xfm.git && cd xfm && mkdir .build && cd .build && cmake .. && make" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) @@ -47,7 +47,7 @@ if( BUILD_DOC ) else() message( "+++ found xfm in the build environment" ) endif() - + endif() # base filenames (with .xfm are input) @@ -85,36 +85,35 @@ if( BUILD_DOC ) rmr_get_srcip.3 ) - # empty list of roff/troff input files we generated + # initialise lists of files we generated set( man3_files ) set( man7_files ) - # mk is so much easier than this -- grumble # for each source, build a specific command that runs tfm to generate the - # troff output. Sed is needed to remove the leading blank that tfm likes - # to insert even if indention is 0. + # troff output as a gzipped file. Sed is needed to remove the leading blank + # that tfm likes to insert even if indention is 0. # foreach( nm IN LISTS man_names ) set( out ${CMAKE_BINARY_DIR}/${nm} ) set( in ${CMAKE_SOURCE_DIR}/doc/src/man/${nm}.xfm ) - add_custom_command( - OUTPUT ${out} + add_custom_command( + OUTPUT ${out}.gz DEPENDS ${in} - COMMAND bash -c "export LIB=${CMAKE_SOURCE_DIR}/doc/src; ${tfm} ${in} stdout | sed 's/^ //' >${out}; ${pfm} ${in} ${out}.ps" + COMMAND bash -c "export LIB=${CMAKE_SOURCE_DIR}/doc/src; ${tfm} ${in} stdout | sed 's/^ //' | gzip >${out}.gz; ${pfm} ${in} ${out}.ps" WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "Building manpage ${out}" VERBATIM ) if( ${out} MATCHES ".*\.3" ) - list( APPEND man3_files ${out} ) + list( APPEND man3_files ${out}.gz ) else() - list( APPEND man7_files ${out} ) + list( APPEND man7_files ${out}.gz ) endif() endforeach() # we must force these to install - # find all of the man pages in build and add them to the deb + # find all of the man pages in build and add them to the package # install( FILES ${man3_files} DESTINATION ${install_man}/man3/ ) install( FILES ${man7_files} DESTINATION ${install_man}/man7/ ) -- 2.16.6