You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
218 lines
7.0 KiB
218 lines
7.0 KiB
#!/bin/bash
|
|
|
|
set -e
|
|
|
|
exec 3>&1 # make stdout available as fd 3 for the result
|
|
exec 1>&2 # redirect all output to stderr for logging
|
|
|
|
echo "BUILD_ID: $BUILD_ID"
|
|
echo "BUILD_NAME: $BUILD_NAME"
|
|
echo "BUILD_JOB_NAME: $BUILD_JOB_NAME"
|
|
echo "BUILD_PIPELINE_NAME: $BUILD_PIPELINE_NAME"
|
|
|
|
# shellcheck disable=1090
|
|
source "$(dirname $0)/common.sh"
|
|
|
|
source=$1
|
|
|
|
if [[ -z "$source" ]]; then
|
|
echo "usage: $0 <path/to/source>"
|
|
exit 1
|
|
fi
|
|
|
|
# for jq
|
|
PATH=/usr/local/bin:$PATH
|
|
|
|
payload=$(mktemp "$TMPDIR/rclone-resource-request.XXXXXX")
|
|
cat > "$payload" <&0
|
|
|
|
load_config "$payload"
|
|
load_files "$payload"
|
|
|
|
rclone_source_directory=$(jq -r '.params.source // ""' < "$payload")
|
|
rclone_destinations=$(jq -r '.params.destination[]? | .dir // ""' < "$payload")
|
|
rclone_link_destination=$(jq -r '.params.linkDestination // ""' < "$payload")
|
|
rclone_link_destination_subdir_file=$(jq -r '.params.linkDestinationSubDir // ""' < "$payload")
|
|
|
|
echo "Source directory: ${rclone_source_directory}"
|
|
echo "Link destination: ${rclone_link_destination}"
|
|
echo "Link destination subdir file: ${rclone_link_destination_subdir_file}"
|
|
|
|
|
|
if [[ -z "$rclone_source_directory" ]]; then
|
|
echo "invalid source directory (missing source)"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "$rclone_destinations" ]]; then
|
|
echo "invalid destination (missing destination)"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Source directory: $rclone_source_directory"
|
|
echo "Destinations:"
|
|
jq -c '.params.destination[] | .dir' < "$payload"
|
|
|
|
local_source_dir="${source}/${rclone_source_directory}"
|
|
ls -alh "${local_source_dir}"
|
|
|
|
echo "Generating SHA256"
|
|
sha256_file="${TMPDIR}/rclone_source.sha256"
|
|
touch "${sha256_file}"
|
|
(find "${local_source_dir}" -type f -exec sha256sum {} \;) | cut -d ' ' -f1 | xargs echo >> "${sha256_file}"
|
|
sha256=$(sha256sum "${sha256_file}" | cut -d' ' -f1)
|
|
echo "sha256:${sha256}"
|
|
|
|
# Source is always the same for each destination
|
|
rclone_source="${local_source_dir}"
|
|
echo "Source: ${rclone_source}"
|
|
|
|
destinations=$(jq -r '.params.destination[] | @base64' < "$payload")
|
|
echo "Using encoded destinations:"
|
|
echo " ${destinations}"
|
|
|
|
file_links=()
|
|
|
|
for destination in $destinations; do
|
|
_jq() {
|
|
echo "${destination}" | base64 -d | jq -r "${1}"
|
|
}
|
|
|
|
rclone_destination=$(_jq '.dir')
|
|
rclone_destination_subdir_file=$(_jq '.subdir // ""')
|
|
rclone_command=$(_jq '.command // "copy"')
|
|
rclone_args=$(_jq '.args // [] | join(" ")')
|
|
rclone_dedupe=$(_jq '.dedupe // "false"')
|
|
rclone_dedupe_mode=$(_jq '.dedupeMode // "newest"')
|
|
rclone_link=$(_jq '.link // "false"')
|
|
rclone_link_find_filter=$(_jq '.linkFilter // "-maxdepth 1 -type f"')
|
|
rclone_allow_failure=$(_jq '.allowFailure // "false"')
|
|
rclone_sleep=$(_jq '.sleep // ""')
|
|
|
|
echo "Destination: $rclone_destination"
|
|
echo "Destination subdir file: ${rclone_destination_subdir_file}"
|
|
echo "Command: ${rclone_command}"
|
|
echo "Additonal args: ${rclone_args}"
|
|
echo "Run dedupe: ${rclone_dedupe}"
|
|
echo "Dedupe mode: ${rclone_dedupe_mode}"
|
|
echo "Generate link: ${rclone_link}"
|
|
echo "Link find filter: ${rclone_link_find_filter}"
|
|
echo "Allow failure: ${rclone_allow_failure}"
|
|
|
|
if [[ "${rclone_allow_failure}" = "true" ]]; then
|
|
set +e
|
|
fi
|
|
|
|
rclone_destination_subdir=""
|
|
if [ -n "$rclone_destination_subdir_file" ]; then
|
|
echo "Looking in ${source}/${rclone_destination_subdir_file} for subdir to use"
|
|
rclone_destination_subdir=$(head -n 1 < "${source}/${rclone_destination_subdir_file}")
|
|
fi
|
|
|
|
if [[ -z "${rclone_destination_subdir}" ]]; then
|
|
rclone_target="${rclone_destination}"
|
|
else
|
|
rclone_target="${rclone_destination}/${rclone_destination_subdir}"
|
|
fi
|
|
echo "Target: ${rclone_target}"
|
|
# shellcheck disable=2086
|
|
rclone ${rclone_command} "${rclone_source}" "${rclone_target}" --size-only --progress --stats=2s ${rclone_args}
|
|
copy_rc=$?
|
|
if [[ -n "${rclone_sleep}" ]]; then
|
|
echo "Sleeping for ${rclone_sleep}"
|
|
sleep ${rclone_sleep}
|
|
fi
|
|
if [[ "${rclone_allow_failure}" = "true" && $copy_rc -gt 0 ]]; then
|
|
echo "Rclone copy failed. Switching to next destination."
|
|
continue
|
|
fi
|
|
|
|
if [[ "$rclone_dedupe" == "true" ]]; then
|
|
echo "Running Dedupe for: ${rclone_target}"
|
|
# shellcheck disable=2086
|
|
rclone dedupe --dedupe-mode "${rclone_dedupe_mode}" "${rclone_target}" --progress --stats=2s ${rclone_args}
|
|
dedupe_rc=$?
|
|
if [[ -n "${rclone_sleep}" ]]; then
|
|
echo "Sleeping for ${rclone_sleep}"
|
|
sleep ${rclone_sleep}
|
|
fi
|
|
if [[ "${rclone_allow_failure}" = "true" && $dedupe_rc -gt 0 ]]; then
|
|
echo "Rclone dedupe failed. Switching to next destination."
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
if [[ "${rclone_link}" == "true" ]]; then
|
|
# shellcheck disable=2086
|
|
filesToLink=$(find "${rclone_source}" ${rclone_link_find_filter})
|
|
echo "Files to link:"
|
|
echo "${filesToLink}"
|
|
for file in ${filesToLink}; do
|
|
remote_file=${file#"$rclone_source/"}
|
|
rclone_target_file="${rclone_target}/${remote_file}"
|
|
echo "Generating Link for: ${rclone_target_file}"
|
|
file_link=$(rclone link "${rclone_target_file}")
|
|
link_rc=$?
|
|
if [[ -n "${rclone_sleep}" ]]; then
|
|
echo "Sleeping for ${rclone_sleep}"
|
|
sleep ${rclone_sleep}
|
|
fi
|
|
if [[ "${rclone_allow_failure}" = "true" && $link_rc -gt 0 ]]; then
|
|
echo "Rclone linking failed. Switching to next target."
|
|
continue
|
|
fi
|
|
file_links+=( "${rclone_target_file}" "${file_link}" )
|
|
done
|
|
fi
|
|
|
|
if [[ "${rclone_allow_failure}" = "true" ]]; then
|
|
set -e
|
|
fi
|
|
|
|
echo "rclone job complete for ${rclone_source} -> ${rclone_target}"
|
|
done
|
|
|
|
# Arithmetic expression to check size of array
|
|
if (( ${#file_links[@]} )); then
|
|
echo "File Links:" "${file_links[@]}"
|
|
# shellcheck disable=2068
|
|
metadata=$(printf '{"name": "%s", "value": "%s"},' ${file_links[@]})
|
|
metadata="[${metadata::-1}]"
|
|
links_json=$( echo -n "{\"links\":${metadata}}" | jq -r . )
|
|
links_json_file="${TMPDIR}/.rclone_links.json"
|
|
echo "$links_json" > "${links_json_file}"
|
|
if [[ -n "${rclone_link_destination}" ]]; then
|
|
rclone_link_destination_subdir=""
|
|
if [ -n "$rclone_link_destination_subdir_file" ]; then
|
|
echo "Looking in ${source}/${rclone_link_destination_subdir_file} for subdir to use"
|
|
rclone_link_destination_subdir=$(head -n 1 < "${source}/${rclone_link_destination_subdir_file}")
|
|
fi
|
|
|
|
if [[ -z "${rclone_link_destination_subdir}" ]]; then
|
|
rclone_link_target="${rclone_link_destination}"
|
|
else
|
|
rclone_link_target="${rclone_link_destination}/${rclone_link_destination_subdir}"
|
|
fi
|
|
echo "Link Target: ${rclone_link_target}"
|
|
rclone copy ${links_json_file} ${rclone_link_target} --progress --stats=2s
|
|
if [[ -n "${rclone_sleep}" ]]; then
|
|
echo "Sleeping for ${rclone_sleep}"
|
|
sleep ${rclone_sleep}
|
|
fi
|
|
fi
|
|
cat "${links_json_file}"
|
|
jq -n "{
|
|
version: {
|
|
build: $( echo -n "${BUILD_ID}" | jq -R .),
|
|
digest: $( echo -n "sha256:${sha256}" | jq -R . )
|
|
},
|
|
metadata: ${metadata}
|
|
}" >&3
|
|
else
|
|
jq -n "{
|
|
version: {
|
|
build: $( echo -n "${BUILD_ID}" | jq -R .),
|
|
digest: $( echo -n "sha256:${sha256}" | jq -R . )
|
|
}
|
|
}" >&3
|
|
fi
|