From 7f0921063549ba0dd8b59be46d169ec6bbb3d275 Mon Sep 17 00:00:00 2001 From: Luciano Silva Date: Fri, 17 Feb 2017 15:47:18 -0500 Subject: [PATCH] Update check script --- assets/check | 48 +++--------- assets/common.sh | 92 ++++++++-------------- test/helpers.sh | 189 +-------------------------------------------- test/itest-out.sh | 100 ------------------------ test/test-check.sh | 72 +++++------------ test/test-regex.sh | 129 ------------------------------- 6 files changed, 64 insertions(+), 566 deletions(-) mode change 100644 => 100755 assets/common.sh delete mode 100755 test/itest-out.sh delete mode 100755 test/test-regex.sh diff --git a/assets/check b/assets/check index 8582448..266461d 100755 --- a/assets/check +++ b/assets/check @@ -1,62 +1,36 @@ #!/bin/bash -set -e +set -ex exec 3>&1 # make stdout available as fd 3 for the result exec 1>&2 # redirect all output to stderr for logging source $(dirname $0)/common.sh -payload=$(mktemp $TMPDIR/artifactory-resource-request.XXXXXX) +payload=$(mktemp $TMPDIR/concoure-curl-resource-request.XXXXXX) cat > $payload <&0 cat $payload -endpoint=$(jq -r '.source.endpoint // ""' < $payload) -regex=$(jq -r '.source.regex // ""' < $payload) -username=$(jq -r '.source.username // ""' < $payload) -password=$(jq -r '.source.password // ""' < $payload) +url=$(jq -r '.source.url // ""' < $payload) +# username=$(jq -r '.source.username // ""' < $payload) +# password=$(jq -r '.source.password // ""' < $payload) skip_ssl_verification=$(jq -r '.source.skip_ssl_verification // ""' < $payload) -repository=$(jq -r '.source.repository // ""' < $payload) -file=$(jq -r '.params.file // ""' < $payload) -folder=$(jq -r '.params.folder // ""' < $payload) -paramRegex=$(jq -r '.params.regex // ""' < $payload) - -version=$(jq -r '.version.version // ""' < $payload) - -if [ -z "$endpoint" ]; then - echo "invalid payload (missing endpoint)" - exit 1 -fi - -if [ -z "$repository" ]; then - echo "invalid payload (missing repository)" +if [ -z "$url" ]; then + echo "invalid payload (missing url)" exit 1 fi -# Building CURL request -args_url="$endpoint/api/storage$repository$folder" - args_security= -[ -n "$username" ] && args_security="-u $username"; -[ -n "$password" ] && args_security="$args_security:$password"; +# [ -n "$username" ] && args_security="-u $username"; +# [ -n "$password" ] && args_security="$args_security:$password"; trueValue="true" [ -n "$skip_ssl_verification" ] && [ "${skip_ssl_verification,,}" = "${trueValue,,}" ] && args_security="$args_security -k"; -if [ -n "$paramRegex" ]; then - echo "overwriting source regex" - regex=$paramRegex -fi - -final_url=$(echo "$args_security" " $args_url") +final_url=$(echo "$args_security" " $url") echo $final_url -if [ -z "$version" ]; then - echo "empty version - return current version" - artifactory_current_version "$final_url" "$regex" >&3 -else - check_version "$final_url" "$regex" "$version" >&3 -fi +check_version "$final_url" >&3 diff --git a/assets/common.sh b/assets/common.sh old mode 100644 new mode 100755 index 1452675..2c48709 --- a/assets/common.sh +++ b/assets/common.sh @@ -1,62 +1,34 @@ - -# Using jq regex so we can support groups -applyRegex_version() { - local regex=$1 - local file=$2 - - jq -n "{ - version: $(echo $file | jq -R .) - }" | jq --arg v "$regex" '.version | capture($v)' | jq -r '.version' - -} - -# retrieve current from artifactory -# e.g url=http://your-host-goes-here:8081/artifactory/api/storage/your-path-goes-here -# regex=ecd-front-(?.*).tar.gz -artifactory_current_version() { - local artifacts_url=$1 - local regex=$2 - - curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[length-1] | {version: .version}]' - -} - -# Return all versions -artifactory_versions() { - local artifacts_url=$1 - local regex=$2 - - curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[] | {version: .version}]' - -} - -# return uri and version of all files -artifactory_files() { - local artifacts_url=$1 - local regex="(?$2)" - - curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[] | {uri: .uri, version: .version}]' - -} - -in_file_with_version() { - local artifacts_url=$1 - local regex="(?$2)" - local version=$3 - - result=$(artifactory_files "$artifacts_url" "$regex") - echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version == $v then $item else empty end)]' - -} - - -# return the list of versions from provided version +set -e + +exec 3>&1 # make stdout available as fd 3 for the result +exec 1>&2 # redirect all output to stderr for logging + +# in_file_with_version() { +# local artifacts_url=$1 +# local regex="(?$2)" +# local version=$3 +# +# result=$(artifactory_files "$artifacts_url" "$regex") +# echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version == $v then $item else empty end)]' +# +# } +# + +# retrieve current file version +# e.g. curl -R -I $1 check_version() { - local artifacts_url=$1 - local regex=$2 - local version=$3 - - result=$(artifactory_versions "$artifacts_url" "$regex") #result=$(curl "$artifacts_url" "$regex") - echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version >= $v then $item else empty end)]' - + # retrieves HTTP header of file URL response + local httpHeader=$(curl -R -I $1 2>&1 | grep 'Last-Modified:') + # Checks if field "Last-Modified" exists in HTTP header and transform it into timestamp string + # if that field is not present, return current timestamp + if [ -z "$httpHeader" ] + then + # echo "Last-Modified information not returned for targeted file. Using current date's timestamp as version number." + local dateString=$(date) + else + # echo "$httpHeader" + local dateString=$(echo "$httpHeader" | sed -e "s/Last-Modified: //" | cut -d',' -f 2) + fi + + date +"%Y%m%d%H%S" -d "$dateString" } diff --git a/test/helpers.sh b/test/helpers.sh index 157b71d..ab77440 100755 --- a/test/helpers.sh +++ b/test/helpers.sh @@ -4,7 +4,7 @@ set -e -u set -o pipefail -export TMPDIR_ROOT=$(mktemp -d /tmp/artifactory-tests.XXXXXX) +export TMPDIR_ROOT=$(mktemp -d /tmp/concoure-curl-resource-tests.XXXXXX) trap "rm -rf $TMPDIR_ROOT" EXIT if [ -d /opt/resource ]; then @@ -14,21 +14,13 @@ else fi run() { - export TMPDIR=$(mktemp -d ${TMPDIR_ROOT}/artifactory-tests.XXXXXX) + export TMPDIR=$(mktemp -d ${TMPDIR_ROOT}/concoure-curl-resource-tests.XXXXXX) - echo -e 'running \e[33m'"$@"$'\e[0m...' + echo -e 'running '"$@"$'\e[0m...' eval "$@" 2>&1 | sed -e 's/^/ /g' echo "" } -find_primary_ip() { - ifconfig | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' -} - -find_docker_host_ip() { - /sbin/ip route | awk '/default/ { print $3 }' -} - create_version_file() { local version=$1 local src=$2 @@ -38,178 +30,3 @@ create_version_file() { echo version/number } - -create_file() { - local src=$1 - local file=$2 - - # Mock the artifact - mkdir $src/build-output - echo "Dummy File" > $src/build-output/$file - touch $src/build-output/$file - - echo "$src/build-output/$file" -} - -# CHECK -check_without_credentials_and_version() { - - local endpoint=$1 - local regex=$2 - local folder=$3 - local src=$4 - - jq -n "{ - source: { - endpoint: $(echo $endpoint | jq -R .), - repository: $(echo $folder | jq -R .), - regex: $(echo $regex | jq -R .) - } - }" | $resource_dir/check "$src" | tee /dev/stderr -} - -# CHECK -check_without_credentials_with_version() { - - local endpoint=$1 - local regex=$2 - local folder=$3 - local version=$4 - local src=$5 - - jq -n "{ - source: { - endpoint: $(echo $endpoint | jq -R .), - repository: $(echo $folder | jq -R .), - regex: $(echo $regex | jq -R .) - }, - version: { version: $(echo $version| jq -R .) - } - }" | $resource_dir/check "$src" | tee /dev/stderr -} - -# CHECK -check_with_credentials_with_version() { - - local endpoint=$1 - local regex=$2 - local username=$3 - local password=$4 - local folder=$5 - local version=$6 - local src=$7 - - jq -n "{ - source: { - endpoint: $(echo $endpoint | jq -R .), - repository: $(echo $folder | jq -R .), - regex: $(echo $regex | jq -R .), - username: $(echo $username | jq -R .), - password: $(echo $password | jq -R .) - }, - version: { version: $(echo $version| jq -R .) - } - }" | $resource_dir/check "$src" | tee /dev/stderr -} - -# IN -in_without_credentials_with_version() { - - local endpoint=$1 - local regex=$2 - local folder=$3 - local version=$4 - local src=$5 - - jq -n "{ - source: { - endpoint: $(echo $endpoint | jq -R .), - repository: $(echo $folder | jq -R .), - regex: $(echo $regex | jq -R .) - }, - version: { version: $(echo $version| jq -R .) - } - }" | $resource_dir/in "$src" | tee /dev/stderr -} - -# IN -in_with_credentials_with_version() { - - local endpoint=$1 - local regex=$2 - local folder=$3 - local version=$4 - local src=$5 - - local username=$6 - local password=$7 - - jq -n "{ - source: { - endpoint: $(echo $endpoint | jq -R .), - username: $(echo $username | jq -R .), - password: $(echo $password | jq -R .), - repository: $(echo $folder | jq -R .), - regex: $(echo $regex | jq -R .) - }, - version: { version: $(echo $version| jq -R .) - } - }" | $resource_dir/in "$src" | tee /dev/stderr -} - -# OUT -deploy_without_credentials() { - - local endpoint=$1 - local regex=$2 - local repository=$3 - local file=$(create_file "$6" "$4") - local version=$5 - local src=$6 - - local version_file=$(create_version_file "$version" "$src") - - jq -n "{ - params: { - file: $(echo $file | jq -R .), - version_file: $(echo $version_file | jq -R .) - }, - source: { - endpoint: $(echo $endpoint | jq -R .), - repository: $(echo $repository | jq -R .), - regex: $(echo $regex | jq -R .) - } - }" | $resource_dir/out "$src" | tee /dev/stderr -} - -# OUT -deploy_with_credentials() { - - local endpoint=$1 - local regex=$2 - local repository=$3 - local file=$(create_file "$6" "$4") - local version=$5 - local src=$6 - - local username=$7 - local password=$8 - - local version_file=$(create_version_file "$version" "$src") - - cat < -# export ART_USER= -# export ART_PWD= -# sample curl commands for artifactory API -# curl -u $ART_USER:$ART_PWD -X PUT "http://host:8081/artifactory/path-to-file" -T ./local-path-to-file -# Artifactory docker image: https://www.jfrog.com/confluence/display/RTF/Running+with+Docker - set -e source $(dirname $0)/helpers.sh +# resource_dir=$(cd $(dirname $0)/../assets && pwd) -it_can_list_releases_from_artifactory() { - - # local local_ip=$(find_docker_host_ip) - #local_ip="localhost" - artifactory_ip=$ART_IP - TMPDIR=/tmp - - local src=$(mktemp -d $TMPDIR/in-src.XXXXXX) - local endpoint="http://${artifactory_ip}:8081/artifactory" - local regex="ecd-front-(?.*).tar.gz" - local folder="/generic/ecd-front" - - check_without_credentials_and_version $endpoint $regex $folder $src - -} - -it_can_list_releases_from_artifactory_with_version() { +# set FILE_URL_WITH_LAST_MODIFIED_INFO with a URL of a file whose HTTP HEADER info provides a Last-Modified entry +# to check it do "curl -I -R " +export FILE_URL_WITH_LAST_MODIFIED_INFO=https://s3-us-west-1.amazonaws.com/lsilva-bpws/PCF_usage/pcf-sandbox-usage-from-2016-09-01-to-2016-09-30_1475771124.json +# set FILE_URL_WITHOUT_LAST_MODIFIED_INFO with a URL of a file whose HTTP HEADER info DOES NOT provide a Last-Modified entry +# to check it do "curl -I -R " +export FILE_URL_WITHOUT_LAST_MODIFIED_INFO=https://raw.githubusercontent.com/pivotalservices/concourse-curl-resource/master/test/data/pivotal-1.0.0.txt - # local local_ip=$(find_docker_host_ip) - #local_ip="localhost" - artifactory_ip=$ART_IP - TMPDIR=/tmp +it_can_get_file_with_last_modified_info() { - local src=$(mktemp -d $TMPDIR/in-src.XXXXXX) - local endpoint="http://${artifactory_ip}:8081/artifactory" - local regex="ecd-front-(?.*).tar.gz" - local folder="/generic/ecd-front" - local version="20161109222826" + echo $resource_dir - check_without_credentials_with_version $endpoint $regex $folder $version $src + jq -n "{ + source: { + url: $(echo $FILE_URL_WITH_LAST_MODIFIED_INFO | jq -R .) + } + }" | $resource_dir/check "$FILE_URL_WITH_LAST_MODIFIED_INFO" | tee /dev/stderr } -it_can_list_releases_from_protected_artifactory_with_version() { - - # local local_ip=$(find_docker_host_ip) - #local_ip="localhost" - artifactory_ip=$ART_IP - TMPDIR=/tmp - - - local src=$(mktemp -d $TMPDIR/in-src.XXXXXX) - local endpoint="http://${artifactory_ip}:8081/artifactory" - local regex="ecd-front-(?.*).tar.gz" - local folder="/generic/ecd-front" - local version="20161109222826" - local username="${ART_USER}" - local password="${ART_PWD}" +it_can_get_file_without_last_modified_info() { - check_with_credentials_with_version $endpoint $regex $username $password $folder $version $src + $resource_dir/check "$FILE_URL_WITHOUT_LAST_MODIFIED_INFO" | tee /dev/stderr } -run it_can_list_releases_from_artifactory -run it_can_list_releases_from_artifactory_with_version -run it_can_list_releases_from_protected_artifactory_with_version +run it_can_get_file_with_last_modified_info +run it_can_get_file_without_last_modified_info diff --git a/test/test-regex.sh b/test/test-regex.sh deleted file mode 100755 index c1edcdd..0000000 --- a/test/test-regex.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash - -# parse file -applyRegex() { - local regex=$1 - local file=$2 - - if [[ $file =~ $regex ]]; - then - version="${BASH_REMATCH[1]}" - echo "${version}" - else - echo "$file doesn't match" >&2 # this could get noisy if there are a lot of non-matching files - exit 1 - fi - echo "${version}" -} - -# Use jq regex since it supports grouping -applyRegex_version() { - local regex=$1 - local file=$2 - - jq -n "{ - version: $(echo $file | jq -R .) - }" | jq --arg v "$regex" '.version | capture($v)' | jq -r '.version' - -} - -# retrieve all versions from artifactory -artifactory_artifacts() { - local artifacts_url=$1 - local regex=$2 - - curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' - -} - -# retrieve current from artifactory -artifactory_current_version() { - local artifacts_url=$1 - local regex=$2 - - curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[length-1] | {version: .version}]' - -} - -# check provided version returning all version -artifactory_versions() { - local artifacts_url=$1 - local regex=$2 - - curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[] | {version: .version}]' - -} - -check_version() { - local artifacts_url=$1 - local regex=$2 - local version=$3 - - result=$(artifactory_versions "$artifacts_url" "$regex") - echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version >= $v then $item else empty end)]' - -} - -# check provided version returning all version -artifactory_files() { - local artifacts_url=$1 - local regex="(?$2)" - - curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[] | {uri: .uri, version: .version}]' - -} - -in_file_with_version() { - local artifacts_url=$1 - local regex="(?$2)" - local version=$3 - - result=$(artifactory_files "$artifacts_url" "$regex") - echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version == $v then $item else empty end)]' - -} - -check_file_with_version() { - local artifacts_url=$1 - local regex=$2 - local version=$3 - - result=$(artifactory_versions "$artifacts_url" "$regex") - echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version == $v then $item else empty end)]' - -} - - -version=$(applyRegex_version "carshare-(?admin|api|customer)-(?.*).tar.gz" "carshare-api-1.0.0-rc.0.tar.gz") -echo "version -> $version" - -url=http://localhost:8081/artifactory/api/storage/UrbanActive/Products/Maven/admin -echo "Testing retrieving artifactis with version group" -echo $(artifactory_artifacts "$url" "carshare-(admin|api|customer)-(?.*).tar.gz") - -echo "Testing retrieving artifactis with module and version group" -echo $(artifactory_artifacts "$url" "carshare-(?admin|api|customer)-(?.*).tar.gz") - -echo "Testing retrieving current version" -echo $(artifactory_current_version "$url" "carshare-(admin|api|customer)-(?.*).tar.gz") - -echo "Testing check version" -result=$(artifactory_versions "$url" "carshare-(admin|api|customer)-(?.*).tar.gz") -echo $result - -result='[ { "version": "1.0.0-rc.0" }, { "version": "1.0.0.2" }, { "version": "1.0.0.3" } ]' - -echo "Testing check by version output" -echo $result | jq '[foreach .[] as $item ([]; $item ; if $item.version >= "1.0.0.2" then $item else empty end)]' - -echo "Testing artifactory files" -result=$(artifactory_files "$url" "carshare-(admin|api|customer)-(?.*).tar.gz") -echo $result - -echo "Testing in with good version" -result=$(in_file_with_version "$url" "carshare-(admin|api|customer)-(?.*).tar.gz" "1.0.0.2") -echo $result - -echo "############### Testing check by version output function" -url="-u admin:password http://192.168.1.224:8081/artifactory/api/storage/libs-snapshot-local/Pivotal" -echo $(check_version "$url" "carshare-(admin|api|customer)-(?.*).tar.gz" "1.0.0.2")