Mirror
/
concourse-curl-resource
mirror of https://github.com/timotto/concourse-curl-resource.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Update check script

master
Luciano Silva 8 years ago
parent
4cc7eae9ec
commit
7f09210635
6 changed files with 64 additions and 566 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 48
      assets/check
  2. 92
      assets/common.sh
  3. 189
      test/helpers.sh
  4. 100
      test/itest-out.sh
  5. 72
      test/test-check.sh
  6. 129
      test/test-regex.sh

48
assets/check
View File

@ -1,62 +1,36 @@
#!/bin/bash #!/bin/bash
set -e
set -ex
exec 3>&1 # make stdout available as fd 3 for the result exec 3>&1 # make stdout available as fd 3 for the result
exec 1>&2 # redirect all output to stderr for logging exec 1>&2 # redirect all output to stderr for logging
source $(dirname $0)/common.sh source $(dirname $0)/common.sh
payload=$(mktemp $TMPDIR/artifactory-resource-request.XXXXXX)
payload=$(mktemp $TMPDIR/concoure-curl-resource-request.XXXXXX)
cat > $payload <&0 cat > $payload <&0
cat $payload cat $payload
endpoint=$(jq -r '.source.endpoint // ""' < $payload)
regex=$(jq -r '.source.regex // ""' < $payload)
username=$(jq -r '.source.username // ""' < $payload)
password=$(jq -r '.source.password // ""' < $payload)
url=$(jq -r '.source.url // ""' < $payload)
# username=$(jq -r '.source.username // ""' < $payload)
# password=$(jq -r '.source.password // ""' < $payload)
skip_ssl_verification=$(jq -r '.source.skip_ssl_verification // ""' < $payload) skip_ssl_verification=$(jq -r '.source.skip_ssl_verification // ""' < $payload)
repository=$(jq -r '.source.repository // ""' < $payload)
file=$(jq -r '.params.file // ""' < $payload)
folder=$(jq -r '.params.folder // ""' < $payload)
paramRegex=$(jq -r '.params.regex // ""' < $payload)
version=$(jq -r '.version.version // ""' < $payload)
if [ -z "$endpoint" ]; then
echo "invalid payload (missing endpoint)"
exit 1
fi
if [ -z "$repository" ]; then
echo "invalid payload (missing repository)"
if [ -z "$url" ]; then
echo "invalid payload (missing url)"
exit 1 exit 1
fi fi
# Building CURL request
args_url="$endpoint/api/storage$repository$folder"
args_security= args_security=
[ -n "$username" ] && args_security="-u $username";
[ -n "$password" ] && args_security="$args_security:$password";
# [ -n "$username" ] && args_security="-u $username";
# [ -n "$password" ] && args_security="$args_security:$password";
trueValue="true" trueValue="true"
[ -n "$skip_ssl_verification" ] && [ "${skip_ssl_verification,,}" = "${trueValue,,}" ] && args_security="$args_security -k"; [ -n "$skip_ssl_verification" ] && [ "${skip_ssl_verification,,}" = "${trueValue,,}" ] && args_security="$args_security -k";
if [ -n "$paramRegex" ]; then
echo "overwriting source regex"
regex=$paramRegex
fi
final_url=$(echo "$args_security" " $args_url")
final_url=$(echo "$args_security" " $url")
echo $final_url echo $final_url
if [ -z "$version" ]; then
echo "empty version - return current version"
artifactory_current_version "$final_url" "$regex" >&3
else
check_version "$final_url" "$regex" "$version" >&3
fi
check_version "$final_url" >&3

92
assets/common.sh
View File

@ -1,62 +1,34 @@
# Using jq regex so we can support groups
applyRegex_version() {
local regex=$1
local file=$2
jq -n "{
version: $(echo $file | jq -R .)
}" | jq --arg v "$regex" '.version | capture($v)' | jq -r '.version'
}
# retrieve current from artifactory
# e.g url=http://your-host-goes-here:8081/artifactory/api/storage/your-path-goes-here
# regex=ecd-front-(?<version>.*).tar.gz
artifactory_current_version() {
local artifacts_url=$1
local regex=$2
curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[length-1] | {version: .version}]'
}
# Return all versions
artifactory_versions() {
local artifacts_url=$1
local regex=$2
curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[] | {version: .version}]'
}
# return uri and version of all files
artifactory_files() {
local artifacts_url=$1
local regex="(?<uri>$2)"
curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[] | {uri: .uri, version: .version}]'
}
in_file_with_version() {
local artifacts_url=$1
local regex="(?<uri>$2)"
local version=$3
result=$(artifactory_files "$artifacts_url" "$regex")
echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version == $v then $item else empty end)]'
}
# return the list of versions from provided version
set -e
exec 3>&1 # make stdout available as fd 3 for the result
exec 1>&2 # redirect all output to stderr for logging
# in_file_with_version() {
# local artifacts_url=$1
# local regex="(?<uri>$2)"
# local version=$3
#
# result=$(artifactory_files "$artifacts_url" "$regex")
# echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version == $v then $item else empty end)]'
#
# }
#
# retrieve current file version
# e.g. curl -R -I $1
check_version() { check_version() {
local artifacts_url=$1
local regex=$2
local version=$3
result=$(artifactory_versions "$artifacts_url" "$regex") #result=$(curl "$artifacts_url" "$regex")
echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version >= $v then $item else empty end)]'
# retrieves HTTP header of file URL response
local httpHeader=$(curl -R -I $1 2>&1 | grep 'Last-Modified:')
# Checks if field "Last-Modified" exists in HTTP header and transform it into timestamp string
# if that field is not present, return current timestamp
if [ -z "$httpHeader" ]
then
# echo "Last-Modified information not returned for targeted file. Using current date's timestamp as version number."
local dateString=$(date)
else
# echo "$httpHeader"
local dateString=$(echo "$httpHeader" | sed -e "s/Last-Modified: //" | cut -d',' -f 2)
fi
date +"%Y%m%d%H%S" -d "$dateString"
} }

189
test/helpers.sh
View File

@ -4,7 +4,7 @@ set -e -u
set -o pipefail set -o pipefail
export TMPDIR_ROOT=$(mktemp -d /tmp/artifactory-tests.XXXXXX)
export TMPDIR_ROOT=$(mktemp -d /tmp/concoure-curl-resource-tests.XXXXXX)
trap "rm -rf $TMPDIR_ROOT" EXIT trap "rm -rf $TMPDIR_ROOT" EXIT
if [ -d /opt/resource ]; then if [ -d /opt/resource ]; then
@ -14,21 +14,13 @@ else
fi fi
run() { run() {
export TMPDIR=$(mktemp -d ${TMPDIR_ROOT}/artifactory-tests.XXXXXX)
export TMPDIR=$(mktemp -d ${TMPDIR_ROOT}/concoure-curl-resource-tests.XXXXXX)
echo -e 'running \e[33m'"$@"$'\e[0m...'
echo -e 'running '"$@"$'\e[0m...'
eval "$@" 2>&1 | sed -e 's/^/ /g' eval "$@" 2>&1 | sed -e 's/^/ /g'
echo "" echo ""
} }
find_primary_ip() {
ifconfig | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p'
}
find_docker_host_ip() {
/sbin/ip route | awk '/default/ { print $3 }'
}
create_version_file() { create_version_file() {
local version=$1 local version=$1
local src=$2 local src=$2
@ -38,178 +30,3 @@ create_version_file() {
echo version/number echo version/number
} }
create_file() {
local src=$1
local file=$2
# Mock the artifact
mkdir $src/build-output
echo "Dummy File" > $src/build-output/$file
touch $src/build-output/$file
echo "$src/build-output/$file"
}
# CHECK
check_without_credentials_and_version() {
local endpoint=$1
local regex=$2
local folder=$3
local src=$4
jq -n "{
source: {
endpoint: $(echo $endpoint | jq -R .),
repository: $(echo $folder | jq -R .),
regex: $(echo $regex | jq -R .)
}
}" | $resource_dir/check "$src" | tee /dev/stderr
}
# CHECK
check_without_credentials_with_version() {
local endpoint=$1
local regex=$2
local folder=$3
local version=$4
local src=$5
jq -n "{
source: {
endpoint: $(echo $endpoint | jq -R .),
repository: $(echo $folder | jq -R .),
regex: $(echo $regex | jq -R .)
},
version: { version: $(echo $version| jq -R .)
}
}" | $resource_dir/check "$src" | tee /dev/stderr
}
# CHECK
check_with_credentials_with_version() {
local endpoint=$1
local regex=$2
local username=$3
local password=$4
local folder=$5
local version=$6
local src=$7
jq -n "{
source: {
endpoint: $(echo $endpoint | jq -R .),
repository: $(echo $folder | jq -R .),
regex: $(echo $regex | jq -R .),
username: $(echo $username | jq -R .),
password: $(echo $password | jq -R .)
},
version: { version: $(echo $version| jq -R .)
}
}" | $resource_dir/check "$src" | tee /dev/stderr
}
# IN
in_without_credentials_with_version() {
local endpoint=$1
local regex=$2
local folder=$3
local version=$4
local src=$5
jq -n "{
source: {
endpoint: $(echo $endpoint | jq -R .),
repository: $(echo $folder | jq -R .),
regex: $(echo $regex | jq -R .)
},
version: { version: $(echo $version| jq -R .)
}
}" | $resource_dir/in "$src" | tee /dev/stderr
}
# IN
in_with_credentials_with_version() {
local endpoint=$1
local regex=$2
local folder=$3
local version=$4
local src=$5
local username=$6
local password=$7
jq -n "{
source: {
endpoint: $(echo $endpoint | jq -R .),
username: $(echo $username | jq -R .),
password: $(echo $password | jq -R .),
repository: $(echo $folder | jq -R .),
regex: $(echo $regex | jq -R .)
},
version: { version: $(echo $version| jq -R .)
}
}" | $resource_dir/in "$src" | tee /dev/stderr
}
# OUT
deploy_without_credentials() {
local endpoint=$1
local regex=$2
local repository=$3
local file=$(create_file "$6" "$4")
local version=$5
local src=$6
local version_file=$(create_version_file "$version" "$src")
jq -n "{
params: {
file: $(echo $file | jq -R .),
version_file: $(echo $version_file | jq -R .)
},
source: {
endpoint: $(echo $endpoint | jq -R .),
repository: $(echo $repository | jq -R .),
regex: $(echo $regex | jq -R .)
}
}" | $resource_dir/out "$src" | tee /dev/stderr
}
# OUT
deploy_with_credentials() {
local endpoint=$1
local regex=$2
local repository=$3
local file=$(create_file "$6" "$4")
local version=$5
local src=$6
local username=$7
local password=$8
local version_file=$(create_version_file "$version" "$src")
cat <<EOF | $resource_dir/out "$src" | tee /dev/stderr
{
"params": {
"file": "$file",
"version_file": "$version_file"
},
"source": {
"endpoint": "$endpoint",
"repository": "$repository",
"username": "$username",
"password": "$password"
}
}
EOF
}

100
test/itest-out.sh
View File

@ -1,100 +0,0 @@
#!/bin/bash
set -e
source $(dirname $0)/helpers.sh
# Export these vars, or let the script prompt you for them
#export MAVEN_RELEASES_URL=http://myrepo.com/repository/releases/
#export MAVEN_SNAPSHOTS_URL=http://myrepo.com/repository/snapshots/
#export MAVEN_REPO_USERNAME=username
#export MAVEN_REPO_PASSWORD=password
#export MAVEN_REPOSITORY_CERT=$(cat /path/to/cert)
if [ -z "$MAVEN_RELEASES_URL" ]; then
echo "Maven Releases Repo URL: "
read -r MAVEN_RELEASES_URL
fi
if [ -z "$MAVEN_SNAPSHOTS_URL" ]; then
echo "Maven Snapshots Repo URL: "
read -r MAVEN_SNAPSHOTS_URL
fi
if [ -z "$MAVEN_REPO_USERNAME" ]; then
echo "Maven Repo Username: "
read -r MAVEN_REPO_USERNAME
fi
if [ -z "$MAVEN_REPO_PASSWORD" ]; then
echo "Maven Repo Password: "
read -r MAVEN_REPO_PASSWORD
fi
it_can_deploy_release_to_manager_without_pom() {
local src=$(mktemp -d $TMPDIR/out-src.XXXXXX)
local url=$MAVEN_RELEASES_URL
local version=1.0.0-rc.0
local username=$MAVEN_REPO_USERNAME
local password=$MAVEN_REPO_PASSWORD
local repository_cert=$MAVEN_REPOSITORY_CERT
deploy_without_pom_with_credentials $url $version $username $password "$repository_cert" $src | jq -e "
.version == {version: $(echo $version | jq -R .)}
"
}
it_can_deploy_snapshot_to_manager_without_pom() {
local src=$(mktemp -d $TMPDIR/out-src.XXXXXX)
local url=$MAVEN_SNAPSHOTS_URL
local version=1.0.0-rc.0-SNAPSHOT
local username=$MAVEN_REPO_USERNAME
local password=$MAVEN_REPO_PASSWORD
local repository_cert=$MAVEN_REPOSITORY_CERT
deploy_without_pom_with_credentials $url $version $username $password "$repository_cert" $src | jq -e "
.version == {version: $(echo $version | jq -R .)}
"
}
it_can_deploy_release_to_manager_with_pom() {
local src=$(mktemp -d $TMPDIR/out-src.XXXXXX)
mkdir $src/project
cp $(dirname $0)/resources/pom-release.xml $src/project/pom.xml
local url=$MAVEN_RELEASES_URL
local pom=$src/project/pom.xml
local version=$(xmllint --xpath "//*[local-name()='project']/*[local-name()='version']/text()" $pom)
local username=$MAVEN_REPO_USERNAME
local password=$MAVEN_REPO_PASSWORD
local repository_cert=$MAVEN_REPOSITORY_CERT
deploy_with_pom_with_credentials $url $pom $username $password "$repository_cert" $src | jq -e "
.version == {version: $(echo $version | jq -R .)}
"
}
it_can_deploy_snapshot_to_manager_with_pom() {
local src=$(mktemp -d $TMPDIR/out-src.XXXXXX)
mkdir $src/project
cp $(dirname $0)/resources/pom-snapshot.xml $src/project/pom.xml
local url=$MAVEN_SNAPSHOTS_URL
local pom=$src/project/pom.xml
local version=$(xmllint --xpath "//*[local-name()='project']/*[local-name()='version']/text()" $pom)
local username=$MAVEN_REPO_USERNAME
local password=$MAVEN_REPO_PASSWORD
local repository_cert=$MAVEN_REPOSITORY_CERT
deploy_with_pom_with_credentials $url $pom $username $password "$repository_cert" $src | jq -e "
.version == {version: $(echo $version | jq -R .)}
"
}
run it_can_deploy_release_to_manager_without_pom
run it_can_deploy_snapshot_to_manager_without_pom
run it_can_deploy_release_to_manager_with_pom
run it_can_deploy_snapshot_to_manager_with_pom

72
test/test-check.sh
View File

@ -1,70 +1,34 @@
#!/bin/bash #!/bin/bash
# before running these tests, set the following env vars:
# export ART_IP=<ip-or-domain-of-artifactory-server>
# export ART_USER=<artifactory-username>
# export ART_PWD=<artifactory-password>
# sample curl commands for artifactory API
# curl -u $ART_USER:$ART_PWD -X PUT "http://host:8081/artifactory/path-to-file" -T ./local-path-to-file
# Artifactory docker image: https://www.jfrog.com/confluence/display/RTF/Running+with+Docker
set -e set -e
source $(dirname $0)/helpers.sh source $(dirname $0)/helpers.sh
# resource_dir=$(cd $(dirname $0)/../assets && pwd)
it_can_list_releases_from_artifactory() {
# local local_ip=$(find_docker_host_ip)
#local_ip="localhost"
artifactory_ip=$ART_IP
TMPDIR=/tmp
local src=$(mktemp -d $TMPDIR/in-src.XXXXXX)
local endpoint="http://${artifactory_ip}:8081/artifactory"
local regex="ecd-front-(?<version>.*).tar.gz"
local folder="/generic/ecd-front"
check_without_credentials_and_version $endpoint $regex $folder $src
}
it_can_list_releases_from_artifactory_with_version() {
# set FILE_URL_WITH_LAST_MODIFIED_INFO with a URL of a file whose HTTP HEADER info provides a Last-Modified entry
# to check it do "curl -I -R <url>"
export FILE_URL_WITH_LAST_MODIFIED_INFO=https://s3-us-west-1.amazonaws.com/lsilva-bpws/PCF_usage/pcf-sandbox-usage-from-2016-09-01-to-2016-09-30_1475771124.json
# set FILE_URL_WITHOUT_LAST_MODIFIED_INFO with a URL of a file whose HTTP HEADER info DOES NOT provide a Last-Modified entry
# to check it do "curl -I -R <url>"
export FILE_URL_WITHOUT_LAST_MODIFIED_INFO=https://raw.githubusercontent.com/pivotalservices/concourse-curl-resource/master/test/data/pivotal-1.0.0.txt
# local local_ip=$(find_docker_host_ip)
#local_ip="localhost"
artifactory_ip=$ART_IP
TMPDIR=/tmp
it_can_get_file_with_last_modified_info() {
local src=$(mktemp -d $TMPDIR/in-src.XXXXXX)
local endpoint="http://${artifactory_ip}:8081/artifactory"
local regex="ecd-front-(?<version>.*).tar.gz"
local folder="/generic/ecd-front"
local version="20161109222826"
echo $resource_dir
check_without_credentials_with_version $endpoint $regex $folder $version $src
jq -n "{
source: {
url: $(echo $FILE_URL_WITH_LAST_MODIFIED_INFO | jq -R .)
}
}" | $resource_dir/check "$FILE_URL_WITH_LAST_MODIFIED_INFO" | tee /dev/stderr
} }
it_can_list_releases_from_protected_artifactory_with_version() {
# local local_ip=$(find_docker_host_ip)
#local_ip="localhost"
artifactory_ip=$ART_IP
TMPDIR=/tmp
local src=$(mktemp -d $TMPDIR/in-src.XXXXXX)
local endpoint="http://${artifactory_ip}:8081/artifactory"
local regex="ecd-front-(?<version>.*).tar.gz"
local folder="/generic/ecd-front"
local version="20161109222826"
local username="${ART_USER}"
local password="${ART_PWD}"
it_can_get_file_without_last_modified_info() {
check_with_credentials_with_version $endpoint $regex $username $password $folder $version $src
$resource_dir/check "$FILE_URL_WITHOUT_LAST_MODIFIED_INFO" | tee /dev/stderr
} }
run it_can_list_releases_from_artifactory
run it_can_list_releases_from_artifactory_with_version
run it_can_list_releases_from_protected_artifactory_with_version
run it_can_get_file_with_last_modified_info
run it_can_get_file_without_last_modified_info

129
test/test-regex.sh
View File

@ -1,129 +0,0 @@
#!/bin/bash
# parse file
applyRegex() {
local regex=$1
local file=$2
if [[ $file =~ $regex ]];
then
version="${BASH_REMATCH[1]}"
echo "${version}"
else
echo "$file doesn't match" >&2 # this could get noisy if there are a lot of non-matching files
exit 1
fi
echo "${version}"
}
# Use jq regex since it supports grouping
applyRegex_version() {
local regex=$1
local file=$2
jq -n "{
version: $(echo $file | jq -R .)
}" | jq --arg v "$regex" '.version | capture($v)' | jq -r '.version'
}
# retrieve all versions from artifactory
artifactory_artifacts() {
local artifacts_url=$1
local regex=$2
curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)'
}
# retrieve current from artifactory
artifactory_current_version() {
local artifacts_url=$1
local regex=$2
curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[length-1] | {version: .version}]'
}
# check provided version returning all version
artifactory_versions() {
local artifacts_url=$1
local regex=$2
curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[] | {version: .version}]'
}
check_version() {
local artifacts_url=$1
local regex=$2
local version=$3
result=$(artifactory_versions "$artifacts_url" "$regex")
echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version >= $v then $item else empty end)]'
}
# check provided version returning all version
artifactory_files() {
local artifacts_url=$1
local regex="(?<uri>$2)"
curl $1 | jq --arg v "$regex" '[.children[].uri | capture($v)]' | jq 'sort_by(.version)' | jq '[.[] | {uri: .uri, version: .version}]'
}
in_file_with_version() {
local artifacts_url=$1
local regex="(?<uri>$2)"
local version=$3
result=$(artifactory_files "$artifacts_url" "$regex")
echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version == $v then $item else empty end)]'
}
check_file_with_version() {
local artifacts_url=$1
local regex=$2
local version=$3
result=$(artifactory_versions "$artifacts_url" "$regex")
echo $result | jq --arg v "$version" '[foreach .[] as $item ([]; $item ; if $item.version == $v then $item else empty end)]'
}
version=$(applyRegex_version "carshare-(?<module>admin|api|customer)-(?<version>.*).tar.gz" "carshare-api-1.0.0-rc.0.tar.gz")
echo "version -> $version"
url=http://localhost:8081/artifactory/api/storage/UrbanActive/Products/Maven/admin
echo "Testing retrieving artifactis with version group"
echo $(artifactory_artifacts "$url" "carshare-(admin|api|customer)-(?<version>.*).tar.gz")
echo "Testing retrieving artifactis with module and version group"
echo $(artifactory_artifacts "$url" "carshare-(?<module>admin|api|customer)-(?<version>.*).tar.gz")
echo "Testing retrieving current version"
echo $(artifactory_current_version "$url" "carshare-(admin|api|customer)-(?<version>.*).tar.gz")
echo "Testing check version"
result=$(artifactory_versions "$url" "carshare-(admin|api|customer)-(?<version>.*).tar.gz")
echo $result
result='[ { "version": "1.0.0-rc.0" }, { "version": "1.0.0.2" }, { "version": "1.0.0.3" } ]'
echo "Testing check by version output"
echo $result | jq '[foreach .[] as $item ([]; $item ; if $item.version >= "1.0.0.2" then $item else empty end)]'
echo "Testing artifactory files"
result=$(artifactory_files "$url" "carshare-(admin|api|customer)-(?<version>.*).tar.gz")
echo $result
echo "Testing in with good version"
result=$(in_file_with_version "$url" "carshare-(admin|api|customer)-(?<version>.*).tar.gz" "1.0.0.2")
echo $result
echo "############### Testing check by version output function"
url="-u admin:password http://192.168.1.224:8081/artifactory/api/storage/libs-snapshot-local/Pivotal"
echo $(check_version "$url" "carshare-(admin|api|customer)-(?<version>.*).tar.gz" "1.0.0.2")
Write Preview
Loading…
Cancel
Save