concourse-rclone-resource/assets/out

#!/bin/bash

set -e

exec 3>&1 # make stdout available as fd 3 for the result
exec 1>&2 # redirect all output to stderr for logging

echo "BUILD_ID: $BUILD_ID"
echo "BUILD_NAME: $BUILD_NAME"
echo "BUILD_JOB_NAME: $BUILD_JOB_NAME"
echo "BUILD_PIPELINE_NAME: $BUILD_PIPELINE_NAME"

# shellcheck disable=1090
source "$(dirname $0)/common.sh"

source=$1

if [[ -z "$source" ]]; then
  echo "usage: $0 <path/to/source>"
  exit 1
fi

# for jq
PATH=/usr/local/bin:$PATH

payload=$(mktemp "$TMPDIR/rclone-resource-request.XXXXXX")
cat > "$payload" <&0

load_config "$payload"
load_files "$payload"

rclone_source_directory=$(jq -r '.params.source // ""' < "$payload")
rclone_destinations=$(jq -r '.params.destination[]? | .dir // ""' < "$payload")
rclone_link_destination=$(jq -r '.params.linkDestination // ""' < "$payload")
rclone_link_destination_subdir_file=$(jq -r '.params.linkDestinationSubDir // ""' < "$payload")

if [[ -z "$rclone_source_directory" ]]; then
  echo "invalid source directory (missing source)"
  exit 1
fi

if [[ -z "$rclone_destinations" ]]; then
  echo "invalid destination (missing destination)"
  exit 1
fi

echo "Source directory: $rclone_source_directory"
echo "Destinations:"
jq -c '.params.destination[] | .dir' < "$payload"

local_source_dir="${source}/${rclone_source_directory}"
ls -alh "${local_source_dir}"

echo "Generating SHA256"
sha256_file="${TMPDIR}/rclone_source.sha256"
touch "${sha256_file}"
(find "${local_source_dir}" -type f -exec sha256sum {} \;) | cut -d ' ' -f1 | xargs echo >> "${sha256_file}"
sha256=$(sha256sum "${sha256_file}" | cut -d' ' -f1)
echo "sha256:${sha256}"

# Source is always the same for each destination
rclone_source="${local_source_dir}"
echo "Source: ${rclone_source}"

destinations=$(jq -r '.params.destination[] | @base64' < "$payload")
echo "Using encoded destinations:"
echo " ${destinations}"

file_links=()

for destination in $destinations; do
  _jq() {
    echo "${destination}" | base64 -d | jq -r "${1}"
  }

  rclone_destination=$(_jq '.dir')
  rclone_destination_subdir_file=$(_jq '.subdir // ""')
  rclone_command=$(_jq '.command // "copy"')
  rclone_args=$(_jq '.args // [] | join(" ")')
  rclone_dedupe=$(_jq '.dedupe // "false"')
  rclone_dedupe_mode=$(_jq '.dedupeMode // "newest"')
  rclone_link=$(_jq '.link // "false"')
  rclone_link_find_filter=$(_jq '.linkFilter // "-maxdepth 1 -type f"')
  rclone_allow_failure=$(_jq '.allowFailure // "false"')
  rclone_sleep=$(_jq '.sleep // ""')

  echo "Destination: $rclone_destination"
  echo "Destination subdir file: ${rclone_destination_subdir_file}"
  echo "Command: ${rclone_command}"
  echo "Additonal args: ${rclone_args}"
  echo "Run dedupe: ${rclone_dedupe}"
  echo "Dedupe mode: ${rclone_dedupe_mode}"
  echo "Generate link: ${rclone_link}"
  echo "Link find filter: ${rclone_link_find_filter}"
  echo "Allow failure: ${rclone_allow_failure}"

  if [[ "${rclone_allow_failure}" = "true" ]]; then
    set +e
  fi

  rclone_destination_subdir=""
  if [ -n "$rclone_destination_subdir_file" ]; then
    echo "Looking in ${source}/${rclone_destination_subdir_file} for subdir to use"
    rclone_destination_subdir=$(head -n 1 < "${source}/${rclone_destination_subdir_file}")
  fi

  if [[ -z "${rclone_destination_subdir}" ]]; then
    rclone_target="${rclone_destination}"
  else
    rclone_target="${rclone_destination}/${rclone_destination_subdir}"
  fi
  echo "Target: ${rclone_target}"
  # shellcheck disable=2086
  rclone ${rclone_command} "${rclone_source}" "${rclone_target}" --size-only --progress --stats=2s ${rclone_args}
  copy_rc=$?
  if [[ -n "${rclone_sleep}" ]]; then
    echo "Sleeping for ${rclone_sleep}"
    sleep ${rclone_sleep}
  fi
  if [[ "${rclone_allow_failure}" = "true" && $copy_rc -gt 0 ]]; then
    echo "Rclone copy failed. Switching to next destination."
    continue
  fi

  if [[ "$rclone_dedupe" == "true" ]]; then
    echo "Running Dedupe for: ${rclone_target}"
     # shellcheck disable=2086
    rclone dedupe --dedupe-mode "${rclone_dedupe_mode}" "${rclone_target}" --progress --stats=2s ${rclone_args}
    dedupe_rc=$?
    if [[ -n "${rclone_sleep}" ]]; then
      echo "Sleeping for ${rclone_sleep}"
      sleep ${rclone_sleep}
    fi
    if [[ "${rclone_allow_failure}" = "true" && $dedupe_rc -gt 0 ]]; then
      echo "Rclone dedupe failed. Switching to next destination."
      continue
    fi
  fi

  if [[ "${rclone_link}" == "true" ]]; then
    # shellcheck disable=2086
    filesToLink=$(find "${rclone_source}" ${rclone_link_find_filter})
    echo "Files to link:"
    echo "${filesToLink}"
    for file in ${filesToLink}; do
      remote_file=${file#"$rclone_source/"}
      rclone_target_file="${rclone_target}/${remote_file}"
      echo "Generating Link for: ${rclone_target_file}"
      file_link=$(rclone link "${rclone_target_file}")
      link_rc=$?
      if [[ -n "${rclone_sleep}" ]]; then
        echo "Sleeping for ${rclone_sleep}"
        sleep ${rclone_sleep}
      fi
      if [[ "${rclone_allow_failure}" = "true" && $link_rc -gt 0 ]]; then
        echo "Rclone linking failed. Switching to next target."
        continue
      fi
      file_links+=( "${rclone_target_file}" "${file_link}" )
    done
  fi

  if [[ "${rclone_allow_failure}" = "true" ]]; then
    set -e
  fi

  echo "rclone job complete for ${rclone_source} -> ${rclone_target}"
done

# Arithmetic expression to check size of array
if (( ${#file_links[@]} )); then
  echo "File Links:" "${file_links[@]}"
  # shellcheck disable=2068
  metadata=$(printf '{"name": "%s", "value": "%s"},' ${file_links[@]})
  metadata="[${metadata::-1}]"
  links_json=$( echo -n "{\"links\":${metadata}}" | jq -r . )
  links_json_file="${TMPDIR}/.rclone_links.json"
  echo "$links_json" > "${links_json_file}"
  if [[ -n "${rclone_link_destination}" ]]; then
    rclone_link_destination_subdir=""
    if [ -n "$rclone_link_destination_subdir_file" ]; then
      echo "Looking in ${source}/${rclone_link_destination_subdir_file} for subdir to use"
      rclone_link_destination_subdir=$(head -n 1 < "${source}/${rclone_link_destination_subdir_file}")
    fi

    if [[ -z "${rclone_link_destination_subdir}" ]]; then
      rclone_link_target="${rclone_link_destination}"
    else
      rclone_link_target="${rclone_link_destination}/${rclone_link_destination_subdir}"
    fi
    echo "Link Target: ${rclone_link_target}"
    rclone copy ${links_json_file} ${rclone_link_target} --progress --stats=2s
    if [[ -n "${rclone_sleep}" ]]; then
      echo "Sleeping for ${rclone_sleep}"
      sleep ${rclone_sleep}
    fi
  fi
  cat "${links_json_file}"
  jq -n "{
    version: {
      build: $( echo -n "${BUILD_ID}" | jq -R .),
      digest: $( echo -n "sha256:${sha256}" | jq -R . )
    },
    metadata: ${metadata}
  }" >&3
else
  jq -n "{
    version: {
      build: $( echo -n "${BUILD_ID}" | jq -R .),
      digest: $( echo -n "sha256:${sha256}" | jq -R . )
    }
  }" >&3
fi