176 lines
5.3 KiB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. #!/bin/bash
  2. set -e
  3. exec 3>&1 # make stdout available as fd 3 for the result
  4. exec 1>&2 # redirect all output to stderr for logging
  5. echo "BUILD_ID: $BUILD_ID"
  6. echo "BUILD_NAME: $BUILD_NAME"
  7. echo "BUILD_JOB_NAME: $BUILD_JOB_NAME"
  8. echo "BUILD_PIPELINE_NAME: $BUILD_PIPELINE_NAME"
  9. # shellcheck disable=1090
  10. source "$(dirname $0)/common.sh"
  11. source=$1
  12. if [[ -z "$source" ]]; then
  13. echo "usage: $0 <path/to/source>"
  14. exit 1
  15. fi
  16. # for jq
  17. PATH=/usr/local/bin:$PATH
  18. payload=$(mktemp "$TMPDIR/rclone-resource-request.XXXXXX")
  19. cat > "$payload" <&0
  20. load_config "$payload"
  21. load_files "$payload"
  22. rclone_source_directory=$(jq -r '.params.source // ""' < "$payload")
  23. rclone_destinations=$(jq -r '.params.destination[]? | .dir // ""' < "$payload")
  24. if [[ -z "$rclone_source_directory" ]]; then
  25. echo "invalid source directory (missing source)"
  26. exit 1
  27. fi
  28. if [[ -z "$rclone_destinations" ]]; then
  29. echo "invalid destination (missing destination)"
  30. exit 1
  31. fi
  32. echo "Source directory: $rclone_source_directory"
  33. echo "Destinations:"
  34. jq -c '.params.destination[] | .dir' < "$payload"
  35. local_source_dir="${source}/${rclone_source_directory}"
  36. ls -alh "${local_source_dir}"
  37. echo "Generating SHA256"
  38. sha256_file="${TMPDIR}/rclone_source.sha256"
  39. touch "${sha256_file}"
  40. (find "${local_source_dir}" -type f -exec sha256sum {} \;) | cut -d ' ' -f1 | xargs echo >> "${sha256_file}"
  41. sha256=$(sha256sum "${sha256_file}" | cut -d' ' -f1)
  42. echo "sha256:${sha256}"
  43. # Source is always the same for each destination
  44. rclone_source="${local_source_dir}"
  45. echo "Source: ${rclone_source}"
  46. destinations=$(jq -r '.params.destination[] | @base64' < "$payload")
  47. echo "Using encoded destinations:"
  48. echo " ${destinations}"
  49. file_links=()
  50. for destination in $destinations; do
  51. _jq() {
  52. echo "${destination}" | base64 -d | jq -r "${1}"
  53. }
  54. rclone_destination=$(_jq '.dir')
  55. rclone_destination_subdir_file=$(_jq '.subdir // ""')
  56. rclone_command=$(_jq '.command // "copy"')
  57. rclone_args=$(_jq '.args // [] | join(" ")')
  58. rclone_dedupe=$(_jq '.dedupe // "false"')
  59. rclone_dedupe_mode=$(_jq '.dedupeMode // "newest"')
  60. rclone_link=$(_jq '.link // "false"')
  61. rclone_link_find_filter=$(_jq '.linkFilter // "-maxdepth 1 -type f"')
  62. rclone_allow_failure=$(_jq '.allowFailure // "false"')
  63. echo "Destination: $rclone_destination"
  64. echo "Destination subdir file: ${rclone_destination_subdir_file}"
  65. echo "Command: ${rclone_command}"
  66. echo "Additonal args: ${rclone_args}"
  67. echo "Run dedupe: ${rclone_dedupe}"
  68. echo "Dedupe mode: ${rclone_dedupe_mode}"
  69. echo "Generate link: ${rclone_link}"
  70. echo "Link find filter: ${rclone_link_find_filter}"
  71. echo "Allow failure: ${rclone_allow_failure}"
  72. if [[ "${rclone_allow_failure}" = "true" ]]; then
  73. set +e
  74. fi
  75. rclone_destination_subdir=""
  76. if [ -n "$rclone_destination_subdir_file" ]; then
  77. echo "Looking in ${source}/${rclone_destination_subdir_file} for subdir to use"
  78. rclone_destination_subdir=$(head -n 1 < "${source}/${rclone_destination_subdir_file}")
  79. fi
  80. if [[ -z "${rclone_destination_subdir}" ]]; then
  81. rclone_target="${rclone_destination}"
  82. else
  83. rclone_target="${rclone_destination}/${rclone_destination_subdir}"
  84. fi
  85. echo "Target: ${rclone_target}"
  86. # shellcheck disable=2086
  87. rclone ${rclone_command} "${rclone_source}" "${rclone_target}" --size-only --progress --stats=2s ${rclone_args}
  88. copy_rc=$?
  89. if [[ "${rclone_allow_failure}" = "true" && $copy_rc -gt 0 ]]; then
  90. echo "Rclone copy failed. Switching to next destination."
  91. continue
  92. fi
  93. if [[ "$rclone_dedupe" == "true" ]]; then
  94. echo "Running Dedupe for: ${rclone_target}"
  95. # shellcheck disable=2086
  96. rclone dedupe --dedupe-mode "${rclone_dedupe_mode}" "${rclone_target}" --progress --stats=2s ${rclone_args}
  97. dedupe_rc=$?
  98. if [[ "${rclone_allow_failure}" = "true" && $dedupe_rc -gt 0 ]]; then
  99. echo "Rclone dedupe failed. Switching to next destination."
  100. continue
  101. fi
  102. fi
  103. if [[ "${rclone_link}" == "true" ]]; then
  104. # shellcheck disable=2086
  105. filesToLink=$(find "${rclone_source}" ${rclone_link_find_filter})
  106. echo "Files to link:"
  107. echo "${filesToLink}"
  108. for file in ${filesToLink}; do
  109. remote_file=${file#"$rclone_source/"}
  110. rclone_target_file="${rclone_target}/${remote_file}"
  111. echo "Generating Link for: ${rclone_target_file}"
  112. file_link=$(rclone link "${rclone_target_file}")
  113. link_rc=$?
  114. if [[ "${rclone_allow_failure}" = "true" && $link_rc -gt 0 ]]; then
  115. echo "Rclone linking failed. Switching to next target."
  116. continue
  117. fi
  118. file_links+=( "${rclone_target_file}" "${file_link}" )
  119. done
  120. fi
  121. if [[ "${rclone_allow_failure}" = "true" ]]; then
  122. set -e
  123. fi
  124. echo "rclone job complete for ${rclone_source} -> ${rclone_target}"
  125. done
  126. if (( ${#file_links[@]} )); then
  127. echo "File Links:" "${file_links[@]}"
  128. # shellcheck disable=2068
  129. metadata=$(printf '{"name": "%s", "value": "%s"},' ${file_links[@]})
  130. metadata="[${metadata::-1}]"
  131. links_json=$( echo -n "{\"links\":${metadata}}" | jq -r . )
  132. echo "$links_json" > "${TMPDIR}/.rclone_links"
  133. cat "${TMPDIR}/.rclone_links"
  134. jq -n "{
  135. version: {
  136. build: $( echo -n "${BUILD_ID}" | jq -R .),
  137. digest: $( echo -n "sha256:${sha256}" | jq -R . )
  138. },
  139. metadata: ${metadata}
  140. }" >&3
  141. else
  142. jq -n "{
  143. version: {
  144. build: $( echo -n "${BUILD_ID}" | jq -R .),
  145. digest: $( echo -n "sha256:${sha256}" | jq -R . )
  146. }
  147. }" >&3
  148. fi