Update IN scripts

7 years ago · 488dabdbf4
6 changed files with 50 additions and 209 deletions
--- a/assets/check
+++ b/assets/check
@ -1,6 +1,6 @@
 #!/bin/bash

-set -ex
+set -e

 exec 3>&1 # make stdout available as fd 3 for the result
 exec 1>&2 # redirect all output to stderr for logging
--- a/assets/in
+++ b/assets/in
@ -16,69 +16,44 @@ fi

 cd $source

-payload=$(mktemp $TMPDIR/artifactory-resource-request.XXXXXX)
+payload=$(mktemp $TMPDIR/concoure-curl-resource-request.XXXXXX)

 cat > $payload <&0

 cat $payload

-endpoint=$(jq -r '.source.endpoint // ""' < $payload)
-regex=$(jq -r '.source.regex // ""' < $payload)
-username=$(jq -r '.source.username // ""' < $payload)
-password=$(jq -r '.source.password // ""' < $payload)
+url=$(jq -r '.source.url // ""' < $payload)
+filename=$(jq -r '.source.filename // ""' < $payload)
 skip_ssl_verification=$(jq -r '.source.skip_ssl_verification // ""' < $payload)

-repository=$(jq -r '.source.repository // ""' < $payload)
-file=$(jq -r '.params.file // ""' < $payload)
-folder=$(jq -r '.params.folder // ""' < $payload)
-paramRegex=$(jq -r '.params.regex // ""' < $payload)
-
-version=$(jq -r '.version.version // ""' < $payload)
-
-if [ -z "$endpoint" ]; then
-  echo "invalid payload (missing endpoint)"
+if [ -z "$url" ]; then
+  echo "invalid payload (missing url)"
  exit 1
 fi

-if [ -z "$repository" ]; then
-  echo "invalid payload (missing repository)"
-  exit 1
+if [ -z "$filename" ]; then
+  filename="file"
 fi

-if [ -z "$version" ]; then
-  echo "Missing version"
-  exit 1
-fi

 # Building CURL request
-args_url="$endpoint/api/storage$repository$folder"
-
+args_url="$url"
 args_security=

-[ -n "$username" ] && args_security="-u $username";
-[ -n "$password" ] && args_security="$args_security:$password";
+# [ -n "$username" ] && args_security="-u $username";
+# [ -n "$password" ] && args_security="$args_security:$password";
 trueValue="true"
 [ -n "$skip_ssl_verification" ] && [ "${skip_ssl_verification,,}" = "${trueValue,,}" ] && args_security="$args_security -k";

-if [ -n "$paramRegex" ]; then
-  echo "overwriting source regex"
-  regex=$paramRegex
-fi
-
 final_url=$(echo "$args_security" " $args_url")
 echo $final_url

-file_json=$(in_file_with_version "$final_url" "$regex" "$version")
-file=$(echo $file_json | jq -r '.[].uri')
-
-if [ -z "$file" ]; then
-  echo "file for version '$version' not found"
+version=$(check_version "$final_url")
+if [ -z "$version" ]; then
+  echo "Version could not be retrieved"
  exit 1
 fi

-args_url="$endpoint$repository/$file"
-
-echo $args_security "-O" "$args_url"
-curl $args_security "-O" "$args_url"
+curl $args_security "$args_url" "-o" "$filename"

-echo $file_json | jq '.[].version | {version: {version: .}}' >&3
+echo "{'version': {'version': '$version'}}" >&3
--- a/assets/out
+++ b/assets/out
@ -21,63 +21,3 @@ payload=$(mktemp $TMPDIR/artifactory-resource-request.XXXXXX)
 cat > $payload <&0

 cat $payload
-
-endpoint=$(jq -r '.source.endpoint // ""' < $payload)
-regex=$(jq -r '.source.regex // ""' < $payload)
-username=$(jq -r '.source.username // ""' < $payload)
-password=$(jq -r '.source.password // ""' < $payload)
-skip_ssl_verification=$(jq -r '.source.skip_ssl_verification // ""' < $payload)
-
-repository=$(jq -r '.source.repository // ""' < $payload)
-folder=$(jq -r '.params.folder // ""' < $payload)
-file=$(jq -r '.params.file // ""' < $payload)
-paramRegex=$(jq -r '.params.regex // ""' < $payload)
-
-if [ -z "$endpoint" ]; then
-  echo "invalid payload (missing endpoint)"
-  exit 1
-fi
-
-if [ -z "$repository" ]; then
-  echo "invalid payload (missing repository)"
-  exit 1
-fi
-
-if [ -z "$file" ]; then
-  echo "invalid payload (missing file)"
-  exit 1
-fi
-
-abs_file=$(ls $file)
-filename=$(basename "$abs_file")
-
-
-args_url="$endpoint"
-args_url="$args_url$repository"
-
-if [ -n "$folder" ]; then
-	echo "adding parameter folder"
-	args_url="$args_url/$folder"
-fi
-
-args_url="$args_url/$filename"
-
-args_security=
-
-[ -n "$username" ] && args_security="-u $username";
-[ -n "$password" ] && args_security="$args_security:$password";
-trueValue="true"
-[ -n "$skip_ssl_verification" ] && [ "${skip_ssl_verification,,}" = "${trueValue,,}" ] && args_security="$args_security -k";
-
-echo "########## $filename, $file"
-
-echo $args_security "-T $abs_file $args_url "
-curl $args_security "-T$abs_file" "$args_url"
-
-
-echo $file $regex
-version=$(applyRegex_version $regex $filename)
-
-jq -n "{
-  version: {version: $(echo $version | jq -R .)}
-}" >&3
--- a/pipeline.yml
+++ b/pipeline.yml
@ -1,55 +1,20 @@
 resource_types:
- name: artifactory
+- name: curl
  type: docker-image
  source:
-    repository: pivotalservices/artifactory-resource
+    repository: pivotalservices/concourse-curl-resource

 resources:
- name: file-repository
-  type: artifactory
+- name: single-file
+  type: curl
  check_every: 1m
  source:
-    endpoint: http://ARTIFACTORY-HOST-NAME-GOES-HERE:8081/artifactory
-    repository: "/repository-name/sub-folder"
-    regex: "myapp-(?<version>.*).txt"
-    username: YOUR-ARTIFACTORY-USERNAME
-    password: YOUR-ARTIFACTORY-PASSWORD
-
+    url: https://s3-us-west-1.amazonaws.com/lsilva-bpws/PCF_usage/pcf-sandbox-usage-from-2016-09-01-to-2016-09-30_1475771124.json
+    filename: pcf-sandbox.json
 jobs:
- name: build-and-save-to-artifactory
-  serial: true
-  public: true
+- name: run-and-get-file
  plan:
-  - task: build-file
-    config:
-      platform: linux
-      image_resource:
-        type: docker-image
-        source:
-          repository: ubuntu
-          tag: "latest"
-      outputs:
-      - name: build
-      run:
-        path: sh
-        args:
-        - -exc
-        - |
-          export DATESTRING=$(date +"%Y%m%d")
-          echo "This is my file" > ./build/myapp-$(date +"%Y%m%d%H%S").txt
-          find .
-  - put: file-repository
-    params: { file: ./build/myapp-*.txt }
-
- name: trigger-when-new-file-is-added-to-artifactory
-  serial: true
-  public: true
-  plan:
-  - get: file-repository
-    trigger: true
-    passed:
-      - build-and-save-to-artifactory
-  - task: use-new-file
+  - task: run-and-get-file
    config:
      platform: linux
      image_resource:
@ -58,12 +23,10 @@ jobs:
          repository: ubuntu
          tag: "latest"
      inputs:
-      - name: file-repository
+      - single-file
      run:
        path: sh
        args:
        - -exc
        - |
-          export DATESTRING=$(date +"%Y%m%d")
-          ls -la file-repository
-          cat ./file-repository/myapp*.txt
+          find .
--- a/test/test-check.sh
+++ b/test/test-check.sh
@ -3,7 +3,6 @@
 set -e

 source $(dirname $0)/helpers.sh
-# resource_dir=$(cd $(dirname $0)/../assets && pwd)

 # set FILE_URL_WITH_LAST_MODIFIED_INFO with a URL of a file whose HTTP HEADER info provides a Last-Modified entry
 # to check it do "curl -I -R <url>"
@ -26,7 +25,11 @@ it_can_get_file_with_last_modified_info() {

 it_can_get_file_without_last_modified_info() {

-  $resource_dir/check "$FILE_URL_WITHOUT_LAST_MODIFIED_INFO" | tee /dev/stderr
+  jq -n "{
+    source: {
+      url: $(echo $FILE_URL_WITHOUT_LAST_MODIFIED_INFO | jq -R .)
+    }
+  }" | $resource_dir/check "$FILE_URL_WITHOUT_LAST_MODIFIED_INFO" | tee /dev/stderr

 }

--- a/test/test-in.sh
+++ b/test/test-in.sh
@ -1,67 +1,27 @@
 #!/bin/bash
-
 set -e

 source $(dirname $0)/helpers.sh

-it_can_get_version_from_artifactory() {
-
-  # local local_ip=$(find_docker_host_ip)
-  #local_ip="localhost"
-
-  artifactory_ip=$ART_IP
-  TMPDIR=/tmp
-
-  local src=$(mktemp -d $TMPDIR/in-src.XXXXXX)
-  local endpoint="http://${artifactory_ip}:8081/artifactory"
-  local regex="ecd-front-(?<version>.*).tar.gz"
-  local folder="/generic/ecd-front"
-  local version="20161109222826"
-
-  in_without_credentials_with_version $endpoint $regex $folder $version $src
-
-}
-
-it_cant_get_version_from_artifactory() {
-
-  # local local_ip=$(find_docker_host_ip)
-  #local_ip="localhost"
-
-  artifactory_ip=$ART_IP
-  TMPDIR=/tmp
-
-  local src=$(mktemp -d $TMPDIR/in-src.XXXXXX)
-  local endpoint="http://${artifactory_ip}:8081/artifactory"
-  local regex="ecd-front-(?<version>.*).tar.gz"
-  local folder="/generic/ecd-front"
-  local version="NONE"
-
-  in_without_credentials_with_version $endpoint $regex $folder $version $src
-
-}
-
-it_can_get_version_from_artifactory_with_credentials() {
-
-  # local local_ip=$(find_docker_host_ip)
-  #local_ip="localhost"
-
-  artifactory_ip=$ART_IP
-  TMPDIR=/tmp
-
-  local src=$(mktemp -d $TMPDIR/in-src.XXXXXX)
-  local endpoint="http://${artifactory_ip}:8081/artifactory"
-  local regex="ecd-front-(?<version>.*).tar.gz"
-  local folder="/generic/ecd-front"
-  local version="20161109222826"
-  local username="${ART_USER}"
-  local password="${ART_PWD}"
-
-  in_with_credentials_with_version $endpoint $regex $folder $version $src $username $password
+export src=$(mktemp -d /tmp/in-src.XXXXXX)
+
+# set FILE_URL_WITH_LAST_MODIFIED_INFO with a URL of a file whose HTTP HEADER info provides a Last-Modified entry
+# to check it do "curl -I -R <url>"
+export FILE_URL_WITH_LAST_MODIFIED_INFO=https://s3-us-west-1.amazonaws.com/lsilva-bpws/PCF_usage/pcf-sandbox-usage-from-2016-09-01-to-2016-09-30_1475771124.json
+export FILE_NAME_1=pcf-sandbox.json
+# set FILE_URL_WITHOUT_LAST_MODIFIED_INFO with a URL of a file whose HTTP HEADER info DOES NOT provide a Last-Modified entry
+# to check it do "curl -I -R <url>"
+export FILE_URL_WITHOUT_LAST_MODIFIED_INFO=https://raw.githubusercontent.com/pivotalservices/concourse-curl-resource/master/test/data/pivotal-1.0.0.txt
+export FILE_NAME_2=ivotal-1.0.0.txt
+
+it_can_get_file_with_date_info() {
+  jq -n "{
+    source: {
+      url: $(echo $FILE_URL_WITH_LAST_MODIFIED_INFO | jq -R .),
+      filename: $(echo $FILE_NAME_1 | jq -R .)
+    }
+  }" | $resource_dir/in "$src" | tee /dev/stderr

 }

-#run it_can_get_version_from_artifactory
-run it_can_get_version_from_artifactory_with_credentials
-
-# check for exit code > 0
-#run it_cant_get_version_from_artifactory
+run it_can_get_file_with_date_info