__print_args() {
echo "Args: remote|remote-remove docker|kube --env-file <environment-filename> [release] [auto-clean] [--stop-at-error] "
echo " [--ricsim-prefix <prefix> ] [--use-local-image <app-nam>+] [--use-snapshot-image <app-nam>+]"
- echo " [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+]"
+ echo " [--use-staging-image <app-nam>+] [--use-release-image <app-nam>+] [--image-repo <repo-address]"
+ echo " [--cluster-timeout <timeout-in seconds>]"
}
if [ $# -eq 1 ] && [ "$1" == "help" ]; then
echo "--use-snapshot-image - The script will use images from the nexus snapshot repo for the supplied apps, space separated list of app short names"
echo "--use-staging-image - The script will use images from the nexus staging repo for the supplied apps, space separated list of app short names"
echo "--use-release-image - The script will use images from the nexus release repo for the supplied apps, space separated list of app short names"
+ echo "--image-repo - Url to optional image repo. Only locally built images will be re-tagged and pushed to this repo"
+ echo "--cluster-timeout - Optional timeout for cluster where it takes time to obtain external ip/host-name. Timeout in seconds. "
echo ""
echo "List of app short names supported: "$APP_SHORT_NAMES
exit 0
TIMER_MEASUREMENTS=".timer_measurement.txt"
echo -e "Activity \t Duration" > $TIMER_MEASUREMENTS
+# If this is set, all used images will be re-tagged and pushed to this repo before any
+IMAGE_REPO_ADR=""
+CLUSTER_TIME_OUT=0
echo "-------------------------------------------------------------------------------------------------"
echo "----------------------------------- Test case: "$ATC
fi
fi
fi
+ if [ $paramerror -eq 0 ]; then
+ if [ "$1" == "--image-repo" ]; then
+ shift;
+ IMAGE_REPO_ADR=$1
+ if [ -z "$1" ]; then
+ paramerror=1
+ if [ -z "$paramerror_str" ]; then
+ paramerror_str="No image repo url found for : '--image-repo'"
+ fi
+ else
+ echo "Option set - Image repo url: "$1
+ shift;
+ foundparm=0
+ fi
+ fi
+ fi
+ if [ $paramerror -eq 0 ]; then
+ if [ "$1" == "--cluster-timeout" ]; then
+ shift;
+ CLUSTER_TIME_OUT=$1
+ if [ -z "$1" ]; then
+ paramerror=1
+ if [ -z "$paramerror_str" ]; then
+ paramerror_str="No timeout value found for : '--cluster-timeout'"
+ fi
+ else
+ #Check if positive int
+ case ${CLUSTER_TIME_OUT#[+]} in
+ *[!0-9]* | '')
+ paramerror=1
+ if [ -z "$paramerror_str" ]; then
+ paramerror_str="Value for '--cluster-timeout' not an int : "$CLUSTER_TIME_OUT
+ fi
+ ;;
+ * ) ;; # Ok
+ esac
+ echo "Option set - Cluster timeout: "$1
+ shift;
+ foundparm=0
+ fi
+ fi
+ fi
done
echo ""
# Check if image env var is set and if so export the env var with image to use (used by docker compose files)
# arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name>
__check_and_create_image_var() {
+
if [ $# -ne 6 ]; then
echo "Expected arg: <app-short-name> <target-variable-name> <image-variable-name> <image-tag-variable-name> <tag-suffix> <image name>"
((IMAGE_ERR++))
tmptag=$4"_"$5
tag="${!tmptag}"
+ optional_image_repo_target=""
+
if [ -z $image ]; then
__check_ignore_image $1
if [ $? -eq 0 ]; then
echo ""
tmp=$tmp"<no-image>\t"
else
+
+ optional_image_repo_target=$image
+
#Add repo depending on image type
if [ "$5" == "REMOTE_RELEASE" ]; then
image=$NEXUS_RELEASE_REPO$image
tmp=$tmp"\t"$5
echo -e "$tmp" >> $image_list_file
#Export the env var
- export "${2}"=$image":"$tag
+ export "${2}"=$image":"$tag #Note, this var may be set to the value of the target value below in __check_and_pull_image
+ if [ ! -z "$IMAGE_REPO_ADR" ] && [ $5 == "LOCAL" ]; then # Only push local images if repo is given
+ export "${2}_SOURCE"=$image":"$tag #Var to keep the actual source image
+ export "${2}_TARGET"=$IMAGE_REPO_ADR"/"$optional_image_repo_target":"$tag #Create image + tag for optional image repo - pushed later if needed
+ else
+ export "${2}_SOURCE"=""
+ export "${2}_TARGET"=""
+ fi
}
# Check if app uses image included in this test run
return 0
}
+# Function to re-tag and image and push to another image repo
+__retag_and_push_image() {
+ if [ ! -z "$IMAGE_REPO_ADR" ]; then
+ source_image="${!1}"
+ trg_var_name=$1_"TARGET" # This var is created in func __check_and_create_image_var
+ target_image="${!trg_var_name}"
+ echo -ne " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD}${SAMELINE}"
+ tmp=$(docker image tag $source_image ${target_image} )
+ if [ $? -ne 0 ]; then
+ docker stop $tmp &> ./tmp/.dockererr
+ ((IMAGE_ERR++))
+ echo ""
+ echo -e " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD} - ${RED}Failed${ERED}"
+ cat ./tmp/.dockererr
+ return 1
+ else
+ echo -e " Attempt to re-tag image to: ${BOLD}${target_image}${EBOLD} - ${GREEN}OK${EGREEN}"
+ fi
+ echo -ne " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD}${SAMELINE}"
+ tmp=$(docker push ${target_image} )
+ if [ $? -ne 0 ]; then
+ docker stop $tmp &> ./tmp/.dockererr
+ ((IMAGE_ERR++))
+ echo ""
+ echo -e " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD} - ${RED}Failed${ERED}"
+ cat ./tmp/.dockererr
+ return 1
+ else
+ echo -e " Attempt to push re-tagged image: ${BOLD}${target_image}${EBOLD} - ${GREEN}OK${EGREEN}"
+ fi
+ export "${1}"=$target_image
+ fi
+ return 0
+}
+
#Function to check if image exist and stop+remove the container+pull new images as needed
-#args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag>
+#args <script-start-arg> <descriptive-image-name> <container-base-name> <image-with-tag-var-name>
__check_and_pull_image() {
- echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$4$EBOLD"
+ source_image="${!4}"
+
+ echo -e " Checking $BOLD$2$EBOLD container(s) with basename: $BOLD$3$EBOLD using image: $BOLD$source_image$EBOLD"
format_string="\"{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\""
- tmp_im=$(docker images --format $format_string ${4})
+ tmp_im=$(docker images --format $format_string $source_image)
if [ $1 == "local" ]; then
if [ -z "$tmp_im" ]; then
- echo -e " "$2" (local image): \033[1m"$4"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
+ echo -e " "$2" (local image): \033[1m"$source_image"\033[0m $RED does not exist in local registry, need to be built (or manually pulled)"$ERED
((IMAGE_ERR++))
return 1
else
- echo -e " "$2" (local image): \033[1m"$4"\033[0m "$GREEN"OK"$EGREEN
+ echo -e " "$2" (local image): \033[1m"$source_image"\033[0m "$GREEN"OK"$EGREEN
fi
elif [ $1 == "remote" ] || [ $1 == "remote-remove" ]; then
if [ $1 == "remote-remove" ]; then
fi
if [ -z "$tmp_im" ]; then
echo -ne " Pulling image${SAMELINE}"
- out=$(docker pull $4)
+ out=$(docker pull $source_image)
if [ $? -ne 0 ]; then
echo ""
echo -e " Pulling image -$RED could not be pulled"$ERED
echo -e " Pulling image -$GREEN OK $EGREEN(exists in local repository)"
fi
fi
- return 0
+
+ __retag_and_push_image $4
+
+ return $?
}
setup_testenvironment() {
# The following sequence pull the configured images
- echo -e $BOLD"Pulling configured images, if needed"$EBOLD
- for imagename in $APP_SHORT_NAMES; do
- __check_included_image $imagename
- incl=$?
- __check_project_image $imagename
- proj=$?
- if [ $incl -eq 0 ]; then
- if [ $proj -eq 0 ]; then
- START_ARG_MOD=$START_ARG
- __check_image_local_override $imagename
- if [ $? -eq 1 ]; then
- START_ARG_MOD="local"
+ echo -e $BOLD"Pulling configured images, if needed"$EBOLD
+ if [ ! -z "$IMAGE_REPO_ADR" ]; then
+ echo -e $YELLOW" Excluding all remote image check/pull when running with image repo: $IMAGE_REPO_ADR"$EYELLOW
+ else
+ for imagename in $APP_SHORT_NAMES; do
+ __check_included_image $imagename
+ incl=$?
+ __check_project_image $imagename
+ proj=$?
+ if [ $incl -eq 0 ]; then
+ if [ $proj -eq 0 ]; then
+ START_ARG_MOD=$START_ARG
+ __check_image_local_override $imagename
+ if [ $? -eq 1 ]; then
+ START_ARG_MOD="local"
+ fi
+ else
+ START_ARG_MOD=$START_ARG
+ fi
+ __check_image_local_build $imagename
+ #No pull of images built locally
+ if [ $? -ne 0 ]; then
+ # A function name is created from the app short name
+ # for example app short name 'HTTPPROXY' -> produce the function
+ # name __HTTPPROXY_imagesetup
+ # This function is called and is expected to exist in the imported
+ # file for the httpproxy test functions
+ # The resulting function impl will call '__check_and_pull_image' function
+ # with appropriate parameters
+ function_pointer="__"$imagename"_imagepull"
+ $function_pointer $START_ARG_MOD $START_ARG
fi
else
- START_ARG_MOD=$START_ARG
+ echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW
fi
- __check_image_local_build $imagename
- #No pull of images built locally
- if [ $? -ne 0 ]; then
- # A function name is created from the app short name
- # for example app short name 'HTTPPROXY' -> produce the function
- # name __HTTPPROXY_imagesetup
- # This function is called and is expected to exist in the imported
- # file for the httpproxy test functions
- # The resulting function impl will call '__check_and_pull_image' function
- # with appropriate parameters
- function_pointer="__"$imagename"_imagepull"
- $function_pointer $START_ARG_MOD $START_ARG
- fi
- else
- echo -e $YELLOW" Excluding $imagename image from image check/pull"$EYELLOW
- fi
- done
+ done
+ fi
#Errors in image setting - exit
if [ $IMAGE_ERR -ne 0 ]; then
echo ""
- # Create a table of the images used in the script
- echo -e $BOLD"Local docker registry images used in the this test script"$EBOLD
+ # Create a table of the images used in the script - from local repo
+ echo -e $BOLD"Local docker registry images used in this test script"$EBOLD
docker_tmp_file=./tmp/.docker-images-table
format_string="{{.Repository}}\\t{{.Tag}}\\t{{.CreatedSince}}\\t{{.Size}}\\t{{.CreatedAt}}"
for imagename in $APP_SHORT_NAMES; do
__check_included_image $imagename
if [ $? -eq 0 ]; then
- # A function name is created from the app short name
- # for example app short name 'MR' -> produce the function
- # name __MR_imagebuild
- # This function is called and is expected to exist in the imported
- # file for the mr test functions
- # The resulting function impl shall build the imagee
- function_pointer="__"$imagename"_image_data"
- $function_pointer "$format_string" $docker_tmp_file
+ # Only print image data if image repo is null, or if image repo is set and image is local
+ print_image_data=0
+ if [ -z "$IMAGE_REPO_ADR" ]; then
+ print_image_data=1
+ else
+ __check_image_local_build $imagename
+ if [ $? -eq 0 ]; then
+ print_image_data=1
+ fi
+ fi
+ if [ $print_image_data -eq 1 ]; then
+ # A function name is created from the app short name
+ # for example app short name 'MR' -> produce the function
+ # name __MR_imagebuild
+ # This function is called and is expected to exist in the imported
+ # file for the mr test functions
+ # The resulting function impl shall build the imagee
+ function_pointer="__"$imagename"_image_data"
+ $function_pointer "$format_string" $docker_tmp_file
+ fi
fi
done
-
column -t -s $'\t' $docker_tmp_file | indent1
echo ""
+ if [ ! -z "$IMAGE_REPO_ADR" ]; then
+
+ # Create a table of the images used in the script - from remote repo
+ echo -e $BOLD"Remote repo images used in this test script"$EBOLD
+ echo -e $YELLOW"-- Note: These image will be pulled when the container starts. Images not managed by the test engine --"$EYELLOW
+
+ docker_tmp_file=./tmp/.docker-images-table
+ format_string="{{.Repository}}\\t{{.Tag}}"
+ echo -e "Application\tRepository\tTag" > $docker_tmp_file
+
+ for imagename in $APP_SHORT_NAMES; do
+ __check_included_image $imagename
+ if [ $? -eq 0 ]; then
+ # Only print image data if image repo is null, or if image repo is set and image is local
+ __check_image_local_build $imagename
+ if [ $? -ne 0 ]; then
+ # A function name is created from the app short name
+ # for example app short name 'MR' -> produce the function
+ # name __MR_imagebuild
+ # This function is called and is expected to exist in the imported
+ # file for the mr test functions
+ # The resulting function impl shall build the imagee
+ function_pointer="__"$imagename"_image_data"
+ $function_pointer "$format_string" $docker_tmp_file
+ fi
+ fi
+ done
+
+ column -t -s $'\t' $docker_tmp_file | indent1
+
+ echo ""
+ fi
+
+ if [ $RUNMODE == "KUBE" ]; then
+
+ echo "================================================================================="
+ echo "================================================================================="
+
+ if [ -z "$IMAGE_REPO_ADR" ]; then
+ echo -e $YELLOW" The image pull policy is set to 'Never' - assuming a local image repo is available for all images"$EYELLOW
+ echo -e " This setting only works on single node clusters on the local machine"
+ echo -e " It does not work with multi-node clusters or remote clusters. "
+ export KUBE_IMAGE_PULL_POLICY="Never"
+ else
+ echo -e $YELLOW" The image pull policy is set to 'Always'"$EYELLOW
+ echo -e " This setting work on local clusters, multi-node clusters and remote cluster. "
+ echo -e " Only locally built images are managed. Remote images are always pulled from remote repos"
+ echo -e " Pulling remote snapshot or staging images my in some case result in pulling newer image versions outside the control of the test engine"
+ export KUBE_IMAGE_PULL_POLICY="Always"
+ fi
+ CLUSTER_IP=$(kubectl config view -o jsonpath={.clusters[0].cluster.server} | awk -F[/:] '{print $4}')
+ echo -e $YELLOW" The cluster hostname/ip is: $CLUSTER_IP"$EYELLOW
+
+ echo "================================================================================="
+ echo "================================================================================="
+ echo ""
+ fi
+
echo -e $BOLD"======================================================="$EBOLD
echo -e $BOLD"== Common test setup completed - test script begins =="$EBOLD
echo -e $BOLD"======================================================="$EBOLD
start_timer() {
echo -e $BOLD"INFO(${BASH_LINENO[0]}): "${FUNCNAME[0]}"," $@ $EBOLD
TC_TIMER=$SECONDS
- echo " Timer started"
+ echo " Timer started: $(date)"
}
# Print the value of the time (in seconds)
return 0
}
+# This function runs a kubectl cmd where a single output value is expected, for example get ip with jsonpath filter.
+# The function retries up to the timeout given in the cmd flag '--cluster-timeout'
+# args: <full kubectl cmd with parameters>
+# (Not for test scripts)
+__kube_cmd_with_timeout() {
+ TS_TMP=$(($SECONDS+$CLUSTER_TIME_OUT))
+
+ while true; do
+ kube_cmd_result=$($@)
+ if [ $? -ne 0 ]; then
+ kube_cmd_result=""
+ fi
+ if [ $SECONDS -ge $TS_TMP ] || [ ! -z "$kube_cmd_result" ] ; then
+ echo $kube_cmd_result
+ return 0
+ fi
+ sleep 1
+ done
+}
+
+# This function starts a pod that cleans a the contents of a path mounted as a pvc
+# After this action the pod should terminate
+# This should only be executed when the pod owning the pvc is not running
+# args: <appname> <namespace> <pvc-name> <path-to remove>
+# (Not for test scripts)
+__kube_clean_pvc() {
+
+ export PVC_CLEANER_NAMESPACE=$2
+ export PVC_CLEANER_CLAIMNAME=$3
+ export PVC_CLEANER_RM_PATH=$4
+ input_yaml=$SIM_GROUP"/pvc-cleaner/"pvc-cleaner.yaml
+ output_yaml=$PWD/tmp/$2-pvc-cleaner.yaml
+
+ envsubst < $input_yaml > $output_yaml
+
+ kubectl delete -f $output_yaml #> /dev/null 2>&1 # Delete the previous terminated pod - if existing
+
+ __kube_create_instance pod pvc-cleaner $input_yaml $output_yaml
+ if [ $? -ne 0 ]; then
+ echo $YELLOW" Could not clean pvc for app: $1 - persistent storage not clean - tests may not work"
+ return 1
+ fi
+
+ term_ts=$(($SECONDS+30))
+ while [ $term_ts -gt $SECONDS ]; do
+ pod_status=$(kubectl get pod pvc-cleaner -n $PVC_CLEANER_NAMESPACE --no-headers -o custom-columns=":status.phase")
+ if [ "$pod_status" == "Succeeded" ]; then
+ return 0
+ fi
+ done
+ return 1
+}
+
# This function scales or deletes all resources for app selected by the testcase.
# args: -
# (Not for test scripts)
# Scale prestarted or managed apps
for imagename in $APP_SHORT_NAMES; do
- __check_included_image $imagename
+ # A function name is created from the app short name
+ # for example app short name 'RICMSIM' -> produce the function
+ # name __RICSIM_kube_scale_zero or __RICSIM_kube_scale_zero_and_wait
+ # This function is called and is expected to exist in the imported
+ # file for the ricsim test functions
+ # The resulting function impl shall scale the resources to 0
+ # For prestarted apps, the function waits until the resources are 0
+ # For included (not prestated) apps, the scaling is just ordered
+ __check_prestarted_image $imagename
if [ $? -eq 0 ]; then
- # A function name is created from the app short name
- # for example app short name 'RICMSIM' -> produce the function
- # name __RICSIM_kube_scale_zero or __RICSIM_kube_scale_zero_and_wait
- # This function is called and is expected to exist in the imported
- # file for the ricsim test functions
- # The resulting function impl shall scale the resources to 0
- __check_prestarted_image $imagename
+ function_pointer="__"$imagename"_kube_scale_zero_and_wait"
+ echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0"
+ $function_pointer
+ else
+ __check_included_image $imagename
if [ $? -eq 0 ]; then
- function_pointer="__"$imagename"_kube_scale_zero_and_wait"
- else
function_pointer="__"$imagename"_kube_scale_zero"
+ echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0"
+ $function_pointer
fi
- echo -e " Scaling all kube resources for app $BOLD $imagename $EBOLD to 0"
- $function_pointer
fi
done
TSTART=$SECONDS
loop_ctr=0
while (( $TSTART+600 > $SECONDS )); do
- result="$(__do_curl $url)"
+ result="$(__do_curl -m 10 $url)"
if [ $? -eq 0 ]; then
if [ ${#result} -gt 15 ]; then
#If response is too long, truncate
# returns: The return code is 0 for ok and 1 for not ok
__do_curl() {
echo ${FUNCNAME[1]} "line: "${BASH_LINENO[1]} >> $HTTPLOG
- curlString="curl -skw %{http_code} $@"
+ proxyflag=""
if [ $RUNMODE == "KUBE" ]; then
- if [ ! -z "$CLUSTER_KUBE_PROXY_NODEPORT" ]; then
- curlString="curl -skw %{http_code} --proxy http://localhost:$CLUSTER_KUBE_PROXY_NODEPORT $@"
+ if [ ! -z "$KUBE_PROXY_PATH" ]; then
+ proxyflag=" --proxy $KUBE_PROXY_PATH"
fi
fi
+ curlString="curl -skw %{http_code} $proxyflag $@"
echo " CMD: $curlString" >> $HTTPLOG
res=$($curlString)
+ retcode=$?
echo " RESP: $res" >> $HTTPLOG
+ echo " RETCODE: $retcode" >> $HTTPLOG
+ if [ $retcode -ne 0 ]; then
+ echo "<no-response-from-server>"
+ return 1
+ fi
http_code="${res:${#res}-3}"
if [ ${#res} -eq 3 ]; then
if [ $http_code -lt 200 ] || [ $http_code -gt 299 ]; then
Code Review / nonrtric.git / blobdiff