#!/bin/bash set -e exec 3>&1 # make stdout available as fd 3 for the result exec 1>&2 # redirect all output to stderr for logging echo "BUILD_ID: $BUILD_ID" echo "BUILD_NAME: $BUILD_NAME" echo "BUILD_JOB_NAME: $BUILD_JOB_NAME" echo "BUILD_PIPELINE_NAME: $BUILD_PIPELINE_NAME" # shellcheck disable=1090 source "$(dirname $0)/common.sh" source=$1 if [[ -z "$source" ]]; then echo "usage: $0 " exit 1 fi # for jq PATH=/usr/local/bin:$PATH payload=$(mktemp "$TMPDIR/rclone-resource-request.XXXXXX") cat > "$payload" <&0 load_config "$payload" load_files "$payload" rclone_source_directory=$(jq -r '.params.source // ""' < "$payload") rclone_destinations=$(jq -r '.params.destination[]? | .dir // ""' < "$payload") if [[ -z "$rclone_source_directory" ]]; then echo "invalid source directory (missing source)" exit 1 fi if [[ -z "$rclone_destinations" ]]; then echo "invalid destination (missing destination)" exit 1 fi echo "Source directory: $rclone_source_directory" echo "Destinations:" jq -c '.params.destination[] | .dir' < "$payload" local_source_dir="${source}/${rclone_source_directory}" ls -alh "${local_source_dir}" echo "Generating SHA256" sha256_file="${TMPDIR}/rclone_source.sha256" touch "${sha256_file}" (find "${local_source_dir}" -type f -exec sha256sum {} \;) | cut -d ' ' -f1 | xargs echo >> "${sha256_file}" sha256=$(sha256sum "${sha256_file}" | cut -d' ' -f1) echo "sha256:${sha256}" # Source is always the same for each destination rclone_source="${local_source_dir}" echo "Source: ${rclone_source}" rclone_sha1=$(rclone sha1sum "${rclone_source}" | cut -d' ' -f1) echo "rclone sha1:${rclone_sha1}" destinations=$(jq -r '.params.destination[] | @base64' < "$payload") echo "Using encoded destinations:" echo " ${destinations}" file_links=() for destination in $destinations; do _jq() { echo "${destination}" | base64 -d | jq -r "${1}" } rclone_destination=$(_jq '.dir') rclone_destination_subdir_file=$(_jq '.subdir // ""') rclone_dedupe=$(_jq '.dedupe // "false"') rclone_dedupe_mode=$(_jq '.dedupeMode // "newest"') rclone_args=$(_jq '.args // [] | join(" ")') rclone_link=$(_jq '.link // "false"') rclone_link_find_filter=$(_jq '.linkFilter // "-maxdepth 1 -type f"') echo "Destination: $rclone_destination" echo "Destination subdir file: ${rclone_destination_subdir_file}" echo "Run dedupe: ${rclone_dedupe}" echo "Dedupe mode: ${rclone_dedupe_mode}" echo "Additonal args: ${rclone_args}" echo "Generate link: ${rclone_link}" echo "Link find filter: ${rclone_link_find_filter}" rclone_destination_subdir="" if [ -n "$rclone_destination_subdir_file" ]; then echo "Looking in ${source}/${rclone_destination_subdir_file} for subdir to use" rclone_destination_subdir=$(head -n 1 < "${source}/${rclone_destination_subdir_file}") fi rclone_target="${rclone_destination}/${rclone_destination_subdir}" echo "Target: ${rclone_target}" # shellcheck disable=2086 rclone copy "${rclone_source}" "${rclone_target}" --size-only --progress --stats=2s ${rclone_args} if [[ "$rclone_dedupe" == "true" ]]; then echo "Running Dedupe for: ${rclone_target}" # shellcheck disable=2086 rclone dedupe --dedupe-mode "${rclone_dedupe_mode}" "${rclone_target}" --progress --stats=2s ${rclone_args} fi if [[ "${rclone_link}" == "true" ]]; then # shellcheck disable=2086 filesToLink=$(find "${rclone_source}" ${rclone_link_find_filter}) echo "Files to link:" echo "${filesToLink}" for file in ${filesToLink}; do rclone_target_file="${rclone_target}/${file}" echo "Generating Link for: ${rclone_target_file}" file_link=$(rclone link "${rclone_target_file}") file_links+=( "${rclone_target_file}" "${file_link}" ) done fi echo "rclone job complete for ${rclone_source} -> ${rclone_target}" done if (( ${#file_links[@]} )); then echo "File Links:" "${file_links[@]}" # shellcheck disable=2068 metadata=$(printf '{"name": "%s", "value": "%s"},' ${file_links[@]}) metadata=${metadata::-1} links_json=$( echo -n "{\"links\":[${metadata}]}" | jq -r . ) echo "Links: ${links_json}" echo "$links_json" > "${rclone_source_directory}/.rclone/links" jq -n "{ version: { $( echo -n "build:${BUILD_ID}" | jq -R .), digest: $( echo -n "sha256:$sha256" | jq -R . ), metadata: ${links_json} } }" >&3 else jq -n "{ version: { $( echo -n "build:${BUILD_ID}" | jq -R .), digest: $( echo -n "sha256:$sha256" | jq -R . ) } }" >&3 fi