From 488dabdbf4ca1467c7ba2af190117e0b21175703 Mon Sep 17 00:00:00 2001 From: Luciano Silva Date: Fri, 17 Feb 2017 17:03:07 -0500 Subject: [PATCH] Update IN scripts --- assets/check | 2 +- assets/in | 55 +++++++++----------------------- assets/out | 60 ----------------------------------- pipeline.yml | 57 ++++++--------------------------- test/test-check.sh | 7 +++-- test/test-in.sh | 78 +++++++++++----------------------------------- 6 files changed, 50 insertions(+), 209 deletions(-) diff --git a/assets/check b/assets/check index 266461d..dc2b55a 100755 --- a/assets/check +++ b/assets/check @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -e exec 3>&1 # make stdout available as fd 3 for the result exec 1>&2 # redirect all output to stderr for logging diff --git a/assets/in b/assets/in index 3aca4d0..43b5fc0 100755 --- a/assets/in +++ b/assets/in @@ -16,69 +16,44 @@ fi cd $source -payload=$(mktemp $TMPDIR/artifactory-resource-request.XXXXXX) +payload=$(mktemp $TMPDIR/concoure-curl-resource-request.XXXXXX) cat > $payload <&0 cat $payload -endpoint=$(jq -r '.source.endpoint // ""' < $payload) -regex=$(jq -r '.source.regex // ""' < $payload) -username=$(jq -r '.source.username // ""' < $payload) -password=$(jq -r '.source.password // ""' < $payload) +url=$(jq -r '.source.url // ""' < $payload) +filename=$(jq -r '.source.filename // ""' < $payload) skip_ssl_verification=$(jq -r '.source.skip_ssl_verification // ""' < $payload) -repository=$(jq -r '.source.repository // ""' < $payload) -file=$(jq -r '.params.file // ""' < $payload) -folder=$(jq -r '.params.folder // ""' < $payload) -paramRegex=$(jq -r '.params.regex // ""' < $payload) - -version=$(jq -r '.version.version // ""' < $payload) - -if [ -z "$endpoint" ]; then - echo "invalid payload (missing endpoint)" +if [ -z "$url" ]; then + echo "invalid payload (missing url)" exit 1 fi -if [ -z "$repository" ]; then - echo "invalid payload (missing repository)" - exit 1 +if [ -z "$filename" ]; then + filename="file" fi -if [ -z "$version" ]; then - echo "Missing version" - exit 1 -fi # Building CURL request -args_url="$endpoint/api/storage$repository$folder" - +args_url="$url" args_security= -[ -n "$username" ] && args_security="-u $username"; -[ -n "$password" ] && args_security="$args_security:$password"; +# [ -n "$username" ] && args_security="-u $username"; +# [ -n "$password" ] && args_security="$args_security:$password"; trueValue="true" [ -n "$skip_ssl_verification" ] && [ "${skip_ssl_verification,,}" = "${trueValue,,}" ] && args_security="$args_security -k"; -if [ -n "$paramRegex" ]; then - echo "overwriting source regex" - regex=$paramRegex -fi - final_url=$(echo "$args_security" " $args_url") echo $final_url -file_json=$(in_file_with_version "$final_url" "$regex" "$version") -file=$(echo $file_json | jq -r '.[].uri') - -if [ -z "$file" ]; then - echo "file for version '$version' not found" +version=$(check_version "$final_url") +if [ -z "$version" ]; then + echo "Version could not be retrieved" exit 1 fi -args_url="$endpoint$repository/$file" - -echo $args_security "-O" "$args_url" -curl $args_security "-O" "$args_url" +curl $args_security "$args_url" "-o" "$filename" -echo $file_json | jq '.[].version | {version: {version: .}}' >&3 +echo "{'version': {'version': '$version'}}" >&3 diff --git a/assets/out b/assets/out index 23aa730..f8987c1 100755 --- a/assets/out +++ b/assets/out @@ -21,63 +21,3 @@ payload=$(mktemp $TMPDIR/artifactory-resource-request.XXXXXX) cat > $payload <&0 cat $payload - -endpoint=$(jq -r '.source.endpoint // ""' < $payload) -regex=$(jq -r '.source.regex // ""' < $payload) -username=$(jq -r '.source.username // ""' < $payload) -password=$(jq -r '.source.password // ""' < $payload) -skip_ssl_verification=$(jq -r '.source.skip_ssl_verification // ""' < $payload) - -repository=$(jq -r '.source.repository // ""' < $payload) -folder=$(jq -r '.params.folder // ""' < $payload) -file=$(jq -r '.params.file // ""' < $payload) -paramRegex=$(jq -r '.params.regex // ""' < $payload) - -if [ -z "$endpoint" ]; then - echo "invalid payload (missing endpoint)" - exit 1 -fi - -if [ -z "$repository" ]; then - echo "invalid payload (missing repository)" - exit 1 -fi - -if [ -z "$file" ]; then - echo "invalid payload (missing file)" - exit 1 -fi - -abs_file=$(ls $file) -filename=$(basename "$abs_file") - - -args_url="$endpoint" -args_url="$args_url$repository" - -if [ -n "$folder" ]; then - echo "adding parameter folder" - args_url="$args_url/$folder" -fi - -args_url="$args_url/$filename" - -args_security= - -[ -n "$username" ] && args_security="-u $username"; -[ -n "$password" ] && args_security="$args_security:$password"; -trueValue="true" -[ -n "$skip_ssl_verification" ] && [ "${skip_ssl_verification,,}" = "${trueValue,,}" ] && args_security="$args_security -k"; - -echo "########## $filename, $file" - -echo $args_security "-T $abs_file $args_url " -curl $args_security "-T$abs_file" "$args_url" - - -echo $file $regex -version=$(applyRegex_version $regex $filename) - -jq -n "{ - version: {version: $(echo $version | jq -R .)} -}" >&3 diff --git a/pipeline.yml b/pipeline.yml index 6e5a3e7..69191cb 100644 --- a/pipeline.yml +++ b/pipeline.yml @@ -1,55 +1,20 @@ resource_types: -- name: artifactory +- name: curl type: docker-image source: - repository: pivotalservices/artifactory-resource + repository: pivotalservices/concourse-curl-resource resources: -- name: file-repository - type: artifactory +- name: single-file + type: curl check_every: 1m source: - endpoint: http://ARTIFACTORY-HOST-NAME-GOES-HERE:8081/artifactory - repository: "/repository-name/sub-folder" - regex: "myapp-(?.*).txt" - username: YOUR-ARTIFACTORY-USERNAME - password: YOUR-ARTIFACTORY-PASSWORD - + url: https://s3-us-west-1.amazonaws.com/lsilva-bpws/PCF_usage/pcf-sandbox-usage-from-2016-09-01-to-2016-09-30_1475771124.json + filename: pcf-sandbox.json jobs: -- name: build-and-save-to-artifactory - serial: true - public: true +- name: run-and-get-file plan: - - task: build-file - config: - platform: linux - image_resource: - type: docker-image - source: - repository: ubuntu - tag: "latest" - outputs: - - name: build - run: - path: sh - args: - - -exc - - | - export DATESTRING=$(date +"%Y%m%d") - echo "This is my file" > ./build/myapp-$(date +"%Y%m%d%H%S").txt - find . - - put: file-repository - params: { file: ./build/myapp-*.txt } - -- name: trigger-when-new-file-is-added-to-artifactory - serial: true - public: true - plan: - - get: file-repository - trigger: true - passed: - - build-and-save-to-artifactory - - task: use-new-file + - task: run-and-get-file config: platform: linux image_resource: @@ -58,12 +23,10 @@ jobs: repository: ubuntu tag: "latest" inputs: - - name: file-repository + - single-file run: path: sh args: - -exc - | - export DATESTRING=$(date +"%Y%m%d") - ls -la file-repository - cat ./file-repository/myapp*.txt + find . diff --git a/test/test-check.sh b/test/test-check.sh index 1e34b46..8d72c4b 100755 --- a/test/test-check.sh +++ b/test/test-check.sh @@ -3,7 +3,6 @@ set -e source $(dirname $0)/helpers.sh -# resource_dir=$(cd $(dirname $0)/../assets && pwd) # set FILE_URL_WITH_LAST_MODIFIED_INFO with a URL of a file whose HTTP HEADER info provides a Last-Modified entry # to check it do "curl -I -R " @@ -26,7 +25,11 @@ it_can_get_file_with_last_modified_info() { it_can_get_file_without_last_modified_info() { - $resource_dir/check "$FILE_URL_WITHOUT_LAST_MODIFIED_INFO" | tee /dev/stderr + jq -n "{ + source: { + url: $(echo $FILE_URL_WITHOUT_LAST_MODIFIED_INFO | jq -R .) + } + }" | $resource_dir/check "$FILE_URL_WITHOUT_LAST_MODIFIED_INFO" | tee /dev/stderr } diff --git a/test/test-in.sh b/test/test-in.sh index 496b996..2d3da17 100755 --- a/test/test-in.sh +++ b/test/test-in.sh @@ -1,67 +1,27 @@ #!/bin/bash - set -e source $(dirname $0)/helpers.sh -it_can_get_version_from_artifactory() { - - # local local_ip=$(find_docker_host_ip) - #local_ip="localhost" - - artifactory_ip=$ART_IP - TMPDIR=/tmp - - local src=$(mktemp -d $TMPDIR/in-src.XXXXXX) - local endpoint="http://${artifactory_ip}:8081/artifactory" - local regex="ecd-front-(?.*).tar.gz" - local folder="/generic/ecd-front" - local version="20161109222826" - - in_without_credentials_with_version $endpoint $regex $folder $version $src - -} - -it_cant_get_version_from_artifactory() { - - # local local_ip=$(find_docker_host_ip) - #local_ip="localhost" - - artifactory_ip=$ART_IP - TMPDIR=/tmp - - local src=$(mktemp -d $TMPDIR/in-src.XXXXXX) - local endpoint="http://${artifactory_ip}:8081/artifactory" - local regex="ecd-front-(?.*).tar.gz" - local folder="/generic/ecd-front" - local version="NONE" - - in_without_credentials_with_version $endpoint $regex $folder $version $src - -} - -it_can_get_version_from_artifactory_with_credentials() { - - # local local_ip=$(find_docker_host_ip) - #local_ip="localhost" - - artifactory_ip=$ART_IP - TMPDIR=/tmp - - local src=$(mktemp -d $TMPDIR/in-src.XXXXXX) - local endpoint="http://${artifactory_ip}:8081/artifactory" - local regex="ecd-front-(?.*).tar.gz" - local folder="/generic/ecd-front" - local version="20161109222826" - local username="${ART_USER}" - local password="${ART_PWD}" - - in_with_credentials_with_version $endpoint $regex $folder $version $src $username $password +export src=$(mktemp -d /tmp/in-src.XXXXXX) + +# set FILE_URL_WITH_LAST_MODIFIED_INFO with a URL of a file whose HTTP HEADER info provides a Last-Modified entry +# to check it do "curl -I -R " +export FILE_URL_WITH_LAST_MODIFIED_INFO=https://s3-us-west-1.amazonaws.com/lsilva-bpws/PCF_usage/pcf-sandbox-usage-from-2016-09-01-to-2016-09-30_1475771124.json +export FILE_NAME_1=pcf-sandbox.json +# set FILE_URL_WITHOUT_LAST_MODIFIED_INFO with a URL of a file whose HTTP HEADER info DOES NOT provide a Last-Modified entry +# to check it do "curl -I -R " +export FILE_URL_WITHOUT_LAST_MODIFIED_INFO=https://raw.githubusercontent.com/pivotalservices/concourse-curl-resource/master/test/data/pivotal-1.0.0.txt +export FILE_NAME_2=ivotal-1.0.0.txt + +it_can_get_file_with_date_info() { + jq -n "{ + source: { + url: $(echo $FILE_URL_WITH_LAST_MODIFIED_INFO | jq -R .), + filename: $(echo $FILE_NAME_1 | jq -R .) + } + }" | $resource_dir/in "$src" | tee /dev/stderr } -#run it_can_get_version_from_artifactory -run it_can_get_version_from_artifactory_with_credentials - -# check for exit code > 0 -#run it_cant_get_version_from_artifactory +run it_can_get_file_with_date_info