#!/bin/bash set -e exec 3>&1 # make stdout available as fd 3 for the result exec 1>&2 # redirect all output to stderr for logging echo "BUILD_ID: $BUILD_ID" echo "BUILD_NAME: $BUILD_NAME" echo "BUILD_JOB_NAME: $BUILD_JOB_NAME" echo "BUILD_PIPELINE_NAME: $BUILD_PIPELINE_NAME" # shellcheck disable=1090 source "$(dirname $0)/common.sh" source=$1 if [[ -z "$source" ]]; then echo "usage: $0 " exit 1 fi # for jq PATH=/usr/local/bin:$PATH payload=$(mktemp "$TMPDIR/rclone-resource-request.XXXXXX") cat > "$payload" <&0 load_config "$payload" load_files "$payload" rclone_source_directory=$(jq -r '.params.source // ""' < "$payload") rclone_destinations=$(jq -r '.params.destination[]? | .dir // ""' < "$payload") rclone_link_destination=$(jq -r '.params.linkDestination // ""' < "$payload") rclone_link_destination_subdir_file=$(jq -r '.params.linkDestinationSubDir // ""' < "$payload") echo "Source directory: ${rclone_source_directory}" echo "Link destination: ${rclone_link_destination}" echo "Link destination subdir file: ${rclone_link_destination_subdir_file}" if [[ -z "$rclone_source_directory" ]]; then echo "invalid source directory (missing source)" exit 1 fi if [[ -z "$rclone_destinations" ]]; then echo "invalid destination (missing destination)" exit 1 fi echo "Source directory: $rclone_source_directory" echo "Destinations:" jq -c '.params.destination[] | .dir' < "$payload" local_source_dir="${source}/${rclone_source_directory}" ls -alh "${local_source_dir}" echo "Generating SHA256" sha256_file="${TMPDIR}/rclone_source.sha256" touch "${sha256_file}" (find "${local_source_dir}" -type f -exec sha256sum {} \;) | cut -d ' ' -f1 | xargs echo >> "${sha256_file}" sha256=$(sha256sum "${sha256_file}" | cut -d' ' -f1) echo "sha256:${sha256}" # Source is always the same for each destination rclone_source="${local_source_dir}" echo "Source: ${rclone_source}" destinations=$(jq -r '.params.destination[] | @base64' < "$payload") echo "Using encoded destinations:" echo " ${destinations}" file_links=() for destination in $destinations; do _jq() { echo "${destination}" | base64 -d | jq -r "${1}" } rclone_destination=$(_jq '.dir') rclone_destination_subdir_file=$(_jq '.subdir // ""') rclone_command=$(_jq '.command // "copy"') rclone_args=$(_jq '.args // [] | join(" ")') rclone_dedupe=$(_jq '.dedupe // "false"') rclone_dedupe_mode=$(_jq '.dedupeMode // "newest"') rclone_link=$(_jq '.link // "false"') rclone_link_find_filter=$(_jq '.linkFilter // "-maxdepth 1 -type f"') rclone_allow_failure=$(_jq '.allowFailure // "false"') rclone_sleep=$(_jq '.sleep // ""') echo "Destination: $rclone_destination" echo "Destination subdir file: ${rclone_destination_subdir_file}" echo "Command: ${rclone_command}" echo "Additonal args: ${rclone_args}" echo "Run dedupe: ${rclone_dedupe}" echo "Dedupe mode: ${rclone_dedupe_mode}" echo "Generate link: ${rclone_link}" echo "Link find filter: ${rclone_link_find_filter}" echo "Allow failure: ${rclone_allow_failure}" if [[ "${rclone_allow_failure}" = "true" ]]; then set +e fi rclone_destination_subdir="" if [ -n "$rclone_destination_subdir_file" ]; then echo "Looking in ${source}/${rclone_destination_subdir_file} for subdir to use" rclone_destination_subdir=$(head -n 1 < "${source}/${rclone_destination_subdir_file}") fi if [[ -z "${rclone_destination_subdir}" ]]; then rclone_target="${rclone_destination}" else rclone_target="${rclone_destination}/${rclone_destination_subdir}" fi echo "Target: ${rclone_target}" # shellcheck disable=2086 rclone ${rclone_command} "${rclone_source}" "${rclone_target}" --size-only --progress --stats=2s ${rclone_args} copy_rc=$? if [[ -n "${rclone_sleep}" ]]; then echo "Sleeping for ${rclone_sleep}" sleep ${rclone_sleep} fi if [[ "${rclone_allow_failure}" = "true" && $copy_rc -gt 0 ]]; then echo "Rclone copy failed. Switching to next destination." continue fi if [[ "$rclone_dedupe" == "true" ]]; then echo "Running Dedupe for: ${rclone_target}" # shellcheck disable=2086 rclone dedupe --dedupe-mode "${rclone_dedupe_mode}" "${rclone_target}" --progress --stats=2s ${rclone_args} dedupe_rc=$? if [[ -n "${rclone_sleep}" ]]; then echo "Sleeping for ${rclone_sleep}" sleep ${rclone_sleep} fi if [[ "${rclone_allow_failure}" = "true" && $dedupe_rc -gt 0 ]]; then echo "Rclone dedupe failed. Switching to next destination." continue fi fi if [[ "${rclone_link}" == "true" ]]; then # shellcheck disable=2086 filesToLink=$(find "${rclone_source}" ${rclone_link_find_filter}) echo "Files to link:" echo "${filesToLink}" for file in ${filesToLink}; do remote_file=${file#"$rclone_source/"} rclone_target_file="${rclone_target}/${remote_file}" echo "Generating Link for: ${rclone_target_file}" file_link=$(rclone link "${rclone_target_file}") link_rc=$? if [[ -n "${rclone_sleep}" ]]; then echo "Sleeping for ${rclone_sleep}" sleep ${rclone_sleep} fi if [[ "${rclone_allow_failure}" = "true" && $link_rc -gt 0 ]]; then echo "Rclone linking failed. Switching to next target." continue fi file_links+=( "${rclone_target_file}" "${file_link}" ) done fi if [[ "${rclone_allow_failure}" = "true" ]]; then set -e fi echo "rclone job complete for ${rclone_source} -> ${rclone_target}" done # Arithmetic expression to check size of array if (( ${#file_links[@]} )); then echo "File Links:" "${file_links[@]}" # shellcheck disable=2068 metadata=$(printf '{"name": "%s", "value": "%s"},' ${file_links[@]}) metadata="[${metadata::-1}]" links_json=$( echo -n "{\"links\":${metadata}}" | jq -r . ) links_json_file="${TMPDIR}/.rclone_links.json" echo "$links_json" > "${links_json_file}" if [[ -n "${rclone_link_destination}" ]]; then rclone_link_destination_subdir="" if [ -n "$rclone_link_destination_subdir_file" ]; then echo "Looking in ${source}/${rclone_link_destination_subdir_file} for subdir to use" rclone_link_destination_subdir=$(head -n 1 < "${source}/${rclone_link_destination_subdir_file}") fi if [[ -z "${rclone_link_destination_subdir}" ]]; then rclone_link_target="${rclone_link_destination}" else rclone_link_target="${rclone_link_destination}/${rclone_link_destination_subdir}" fi echo "Link Target: ${rclone_link_target}" rclone copy ${links_json_file} ${rclone_link_target} --progress --stats=2s if [[ -n "${rclone_sleep}" ]]; then echo "Sleeping for ${rclone_sleep}" sleep ${rclone_sleep} fi fi cat "${links_json_file}" jq -n "{ version: { build: $( echo -n "${BUILD_ID}" | jq -R .), digest: $( echo -n "sha256:${sha256}" | jq -R . ) }, metadata: ${metadata} }" >&3 else jq -n "{ version: { build: $( echo -n "${BUILD_ID}" | jq -R .), digest: $( echo -n "sha256:${sha256}" | jq -R . ) } }" >&3 fi