|
#!/usr/bin/env bash |
|
set -euo pipefail |
|
|
|
SCRIPT_NAME="$(basename "$0")" |
|
|
|
CONFIG_FILE="" |
|
VERBOSE=0 |
|
|
|
WORK_DIR="" |
|
RELEASES_DIR="" |
|
CLEANUP_WORK_DIR=false |
|
|
|
log() { |
|
local level="$1"; shift |
|
local ts |
|
ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" |
|
|
|
[[ "$level" == "DEBUG" && "$VERBOSE" -lt 1 ]] && return 0 |
|
printf '%s [%s] %s\n' "$ts" "$level" "$*" >&2 |
|
} |
|
|
|
die() { |
|
log "ERROR" "$*" |
|
exit 1 |
|
} |
|
|
|
usage() { |
|
cat <<EOF |
|
Usage: |
|
$SCRIPT_NAME --config <path> [--verbose] |
|
|
|
Options: |
|
--config <path> Path to Bash-native release mirror config file |
|
--verbose Enable verbose logging. Can be repeated. |
|
--help Show help |
|
|
|
Required commands: |
|
bash, curl, jq, date, mktemp, mkdir, basename, find, sha256sum |
|
EOF |
|
} |
|
|
|
expand_path() { |
|
local path="$1" |
|
case "$path" in |
|
"~") printf '%s\n' "$HOME" ;; |
|
"~/"*) printf '%s/%s\n' "$HOME" "${path#"~/"}" ;; |
|
*) printf '%s\n' "$path" ;; |
|
esac |
|
} |
|
|
|
parse_args() { |
|
while [[ $# -gt 0 ]]; do |
|
case "$1" in |
|
--config) |
|
[[ $# -ge 2 ]] || die "--config requires a path" |
|
CONFIG_FILE="$2" |
|
shift 2 |
|
;; |
|
--verbose) |
|
VERBOSE=$((VERBOSE + 1)) |
|
shift |
|
;; |
|
--help|-h) |
|
usage |
|
exit 0 |
|
;; |
|
*) |
|
die "Unknown argument: $1" |
|
;; |
|
esac |
|
done |
|
} |
|
|
|
require_command() { |
|
local name="$1" |
|
command -v "$name" >/dev/null 2>&1 || die "Required command not found: $name" |
|
} |
|
|
|
get_dest_var() { |
|
local dest_name="$1" |
|
local suffix="$2" |
|
local var_name="DEST_${dest_name}_${suffix}" |
|
|
|
printf '%s' "${!var_name-}" |
|
} |
|
|
|
get_config_var() { |
|
local name="$1" |
|
printf '%s' "${!name-}" |
|
} |
|
|
|
urlencode() { |
|
jq -rn --arg v "$1" '$v|@uri' |
|
} |
|
|
|
json_string() { |
|
jq -Rn --arg v "$1" '$v' |
|
} |
|
|
|
bool_json() { |
|
case "${1:-false}" in |
|
true|TRUE|yes|YES|1) printf 'true' ;; |
|
false|FALSE|no|NO|0|"") printf 'false' ;; |
|
*) die "Invalid boolean value: $1" ;; |
|
esac |
|
} |
|
|
|
http_headers_from_spec() { |
|
# Reads header specs from config variables and expands {{TOKEN}}. |
|
# Header spec supports either: |
|
# "Authorization: Bearer {{TOKEN}}" |
|
# "PRIVATE-TOKEN: {{TOKEN}}" |
|
# It prints null-delimited curl args: -H <header>. |
|
local token="$1"; shift |
|
local header |
|
for header in "$@"; do |
|
[[ -n "$header" ]] || continue |
|
header="${header//'{{TOKEN}}'/$token}" |
|
printf '%s\0%s\0' "-H" "$header" |
|
done |
|
} |
|
|
|
curl_status() { |
|
# Usage: |
|
# curl_status METHOD URL BODY_FILE RESPONSE_FILE CURL_EXTRA_ARGS... |
|
# BODY_FILE may be empty. |
|
local method="$1" |
|
local url="$2" |
|
local body_file="$3" |
|
local response_file="$4" |
|
shift 4 |
|
|
|
local status |
|
local args=(-sS -L -X "$method" -o "$response_file" -w "%{http_code}") |
|
|
|
if [[ -n "$body_file" ]]; then |
|
args+=(--data-binary "@$body_file") |
|
fi |
|
|
|
status="$(curl "${args[@]}" "$@" "$url")" |
|
printf '%s\n' "$status" |
|
} |
|
|
|
load_config() { |
|
[[ -n "$CONFIG_FILE" ]] || die "Missing --config <path>" |
|
[[ -f "$CONFIG_FILE" ]] || die "Config file does not exist: $CONFIG_FILE" |
|
|
|
# shellcheck source=/dev/null |
|
source "$CONFIG_FILE" |
|
|
|
: "${SOURCE_GITHUB_OWNER:?SOURCE_GITHUB_OWNER is required}" |
|
: "${SOURCE_GITHUB_REPO:?SOURCE_GITHUB_REPO is required}" |
|
: "${DESTINATIONS:?DESTINATIONS array is required}" |
|
|
|
SOURCE_GITHUB_API="${SOURCE_GITHUB_API:-https://api.github.com}" |
|
SOURCE_GITHUB_TOKEN="${SOURCE_GITHUB_TOKEN:-}" |
|
SOURCE_GITHUB_TOKEN_HEADER="${SOURCE_GITHUB_TOKEN_HEADER:-Authorization: Bearer {{TOKEN}}}" |
|
SOURCE_RELEASE_SELECTOR="${SOURCE_RELEASE_SELECTOR:-latest}" |
|
SOURCE_RELEASE_TAGS="${SOURCE_RELEASE_TAGS:-}" |
|
SOURCE_INCLUDE_DRAFTS="${SOURCE_INCLUDE_DRAFTS:-false}" |
|
SOURCE_INCLUDE_PRERELEASES="${SOURCE_INCLUDE_PRERELEASES:-true}" |
|
|
|
WORK_MODE="${WORK_MODE:-temp}" |
|
WORK_DIR="${WORK_DIR:-}" |
|
|
|
DOWNLOAD_ASSETS="${DOWNLOAD_ASSETS:-true}" |
|
DOWNLOAD_SOURCE_ARCHIVES="${DOWNLOAD_SOURCE_ARCHIVES:-false}" |
|
SOURCE_ARCHIVE_FORMATS="${SOURCE_ARCHIVE_FORMATS:-tarball zipball}" |
|
|
|
# Vendor modules are expected to be named: module-<vendor>.sh next to the config |
|
|
|
if [[ "$WORK_MODE" != "temp" && "$WORK_MODE" != "persistent" ]]; then |
|
die "WORK_MODE must be either 'temp' or 'persistent'" |
|
fi |
|
|
|
if [[ "$WORK_MODE" == "persistent" && -z "$WORK_DIR" ]]; then |
|
die "WORK_DIR is required when WORK_MODE=persistent" |
|
fi |
|
} |
|
|
|
prepare_workdir() { |
|
if [[ "$WORK_MODE" == "temp" ]]; then |
|
WORK_DIR="$(mktemp -d)" |
|
CLEANUP_WORK_DIR=true |
|
else |
|
WORK_DIR="$(expand_path "$WORK_DIR")" |
|
mkdir -p "$WORK_DIR" |
|
CLEANUP_WORK_DIR=false |
|
fi |
|
|
|
RELEASES_DIR="$WORK_DIR/releases" |
|
mkdir -p "$RELEASES_DIR" |
|
|
|
log "INFO" "Work mode: $WORK_MODE" |
|
log "INFO" "Work directory: $WORK_DIR" |
|
} |
|
|
|
cleanup() { |
|
if [[ "$CLEANUP_WORK_DIR" == "true" && -n "${WORK_DIR:-}" && -d "$WORK_DIR" ]]; then |
|
log "DEBUG" "Cleaning temporary work directory: $WORK_DIR" |
|
rm -rf "$WORK_DIR" |
|
fi |
|
} |
|
|
|
github_headers() { |
|
local token="${SOURCE_GITHUB_TOKEN:-}" |
|
local header="${SOURCE_GITHUB_TOKEN_HEADER:-Authorization: Bearer {{TOKEN}}}" |
|
|
|
printf '%s\0%s\0' "-H" "Accept: application/vnd.github+json" |
|
printf '%s\0%s\0' "-H" "X-GitHub-Api-Version: 2022-11-28" |
|
|
|
if [[ -n "$token" ]]; then |
|
header="${header//'{{TOKEN}}'/$token}" |
|
printf '%s\0%s\0' "-H" "$header" |
|
fi |
|
} |
|
|
|
curl_with_nul_headers() { |
|
# First arg is output file; second arg URL; stdin is null-delimited curl args. |
|
local out_file="$1" |
|
local url="$2" |
|
shift 2 |
|
|
|
local -a extra=() |
|
while IFS= read -r -d '' part; do |
|
extra+=("$part") |
|
done |
|
|
|
curl -sS -L "${extra[@]}" "$@" -o "$out_file" "$url" |
|
} |
|
|
|
github_api_get() { |
|
local path="$1" |
|
local out_file="$2" |
|
local url="${SOURCE_GITHUB_API%/}${path}" |
|
|
|
github_headers | curl_with_nul_headers "$out_file" "$url" |
|
} |
|
|
|
github_download() { |
|
local url="$1" |
|
local out_file="$2" |
|
|
|
if [[ -n "${SOURCE_GITHUB_TOKEN:-}" ]]; then |
|
github_headers | curl_with_nul_headers "$out_file" "$url" |
|
else |
|
curl -sS -L -o "$out_file" "$url" |
|
fi |
|
} |
|
|
|
fetch_source_releases() { |
|
local releases_json="$WORK_DIR/source-releases.json" |
|
|
|
case "$SOURCE_RELEASE_SELECTOR" in |
|
latest) |
|
github_api_get "/repos/$SOURCE_GITHUB_OWNER/$SOURCE_GITHUB_REPO/releases/latest" "$releases_json.one" |
|
jq -s '.' "$releases_json.one" > "$releases_json" |
|
;; |
|
tags) |
|
[[ -n "$SOURCE_RELEASE_TAGS" ]] || die "SOURCE_RELEASE_TAGS is required when SOURCE_RELEASE_SELECTOR=tags" |
|
: > "$releases_json.tmp" |
|
local tag encoded |
|
printf '[' > "$releases_json" |
|
local first=true |
|
for tag in $SOURCE_RELEASE_TAGS; do |
|
encoded="$(urlencode "$tag")" |
|
github_api_get "/repos/$SOURCE_GITHUB_OWNER/$SOURCE_GITHUB_REPO/releases/tags/$encoded" "$releases_json.one" |
|
if [[ "$first" == "true" ]]; then |
|
first=false |
|
else |
|
printf ',' >> "$releases_json" |
|
fi |
|
cat "$releases_json.one" >> "$releases_json" |
|
done |
|
printf ']\n' >> "$releases_json" |
|
;; |
|
all) |
|
github_api_get "/repos/$SOURCE_GITHUB_OWNER/$SOURCE_GITHUB_REPO/releases?per_page=100" "$releases_json" |
|
;; |
|
*) |
|
die "SOURCE_RELEASE_SELECTOR must be one of: latest, tags, all" |
|
;; |
|
esac |
|
|
|
jq -e 'type == "array"' "$releases_json" >/dev/null || die "GitHub releases response is not an array" |
|
printf '%s\n' "$releases_json" |
|
} |
|
|
|
release_is_selected() { |
|
local release_json="$1" |
|
|
|
local draft prerelease include_drafts include_prereleases |
|
draft="$(jq -r '.draft // false' "$release_json")" |
|
prerelease="$(jq -r '.prerelease // false' "$release_json")" |
|
include_drafts="$(bool_json "$SOURCE_INCLUDE_DRAFTS")" |
|
include_prereleases="$(bool_json "$SOURCE_INCLUDE_PRERELEASES")" |
|
|
|
[[ "$draft" == "true" && "$include_drafts" != "true" ]] && return 1 |
|
[[ "$prerelease" == "true" && "$include_prereleases" != "true" ]] && return 1 |
|
return 0 |
|
} |
|
|
|
materialize_release() { |
|
local release_json="$1" |
|
local tag="$2" |
|
|
|
local release_dir="$RELEASES_DIR/$tag" |
|
local assets_dir="$release_dir/assets" |
|
mkdir -p "$assets_dir" |
|
|
|
cp "$release_json" "$release_dir/release.json" |
|
|
|
if [[ "$DOWNLOAD_ASSETS" == "true" ]]; then |
|
local count i name url |
|
count="$(jq '.assets | length' "$release_json")" |
|
|
|
for (( i=0; i<count; i++ )); do |
|
name="$(jq -r ".assets[$i].name" "$release_json")" |
|
url="$(jq -r ".assets[$i].browser_download_url" "$release_json")" |
|
|
|
[[ -n "$name" && "$name" != "null" ]] || die "Release '$tag' has asset without name" |
|
[[ -n "$url" && "$url" != "null" ]] || die "Release '$tag' asset '$name' has no download URL" |
|
|
|
log "INFO" "Downloading GitHub asset: tag=$tag asset=$name" |
|
github_download "$url" "$assets_dir/$name" |
|
done |
|
fi |
|
|
|
if [[ "$DOWNLOAD_SOURCE_ARCHIVES" == "true" ]]; then |
|
local fmt archive_url archive_name |
|
for fmt in $SOURCE_ARCHIVE_FORMATS; do |
|
case "$fmt" in |
|
tarball) |
|
archive_url="$(jq -r '.tarball_url // empty' "$release_json")" |
|
archive_name="${SOURCE_GITHUB_REPO}-${tag}.tar.gz" |
|
;; |
|
zipball) |
|
archive_url="$(jq -r '.zipball_url // empty' "$release_json")" |
|
archive_name="${SOURCE_GITHUB_REPO}-${tag}.zip" |
|
;; |
|
*) |
|
die "Unsupported SOURCE_ARCHIVE_FORMATS value: $fmt" |
|
;; |
|
esac |
|
|
|
[[ -n "$archive_url" ]] || continue |
|
log "INFO" "Downloading GitHub source archive: tag=$tag archive=$archive_name" |
|
github_download "$archive_url" "$assets_dir/$archive_name" |
|
done |
|
fi |
|
|
|
find "$assets_dir" -type f -maxdepth 1 -print0 | sort -z | while IFS= read -r -d '' file; do |
|
sha256sum "$file" |
|
done > "$release_dir/SHA256SUMS" |
|
|
|
printf '%s\n' "$release_dir" |
|
} |
|
|
|
load_vendor_module() { |
|
local vendor="$1" |
|
local module_file |
|
|
|
module_file="$(get_config_var "VENDOR_${vendor}_MODULE")" |
|
module_file="${module_file:-$(dirname "$CONFIG_FILE")/module-${vendor}.sh}" |
|
module_file="$(expand_path "$module_file")" |
|
|
|
[[ -f "$module_file" ]] || die "Vendor module not found for '$vendor': $module_file" |
|
|
|
# shellcheck source=/dev/null |
|
source "$module_file" |
|
|
|
local fn="release_vendor_${vendor}_assert" |
|
declare -F "$fn" >/dev/null || die "Vendor module '$vendor' does not define $fn" |
|
"$fn" |
|
} |
|
|
|
destination_release_exists() { |
|
local dest="$1" |
|
local vendor="$2" |
|
local tag="$3" |
|
local fn="release_vendor_${vendor}_release_exists" |
|
|
|
declare -F "$fn" >/dev/null || die "Vendor module '$vendor' does not define $fn" |
|
"$fn" "$dest" "$tag" |
|
} |
|
|
|
destination_create_release() { |
|
local dest="$1" |
|
local vendor="$2" |
|
local release_dir="$3" |
|
local fn="release_vendor_${vendor}_create_release" |
|
|
|
declare -F "$fn" >/dev/null || die "Vendor module '$vendor' does not define $fn" |
|
"$fn" "$dest" "$release_dir" |
|
} |
|
|
|
destination_upload_assets() { |
|
local dest="$1" |
|
local vendor="$2" |
|
local release_id="$3" |
|
local release_dir="$4" |
|
local fn="release_vendor_${vendor}_upload_assets" |
|
|
|
declare -F "$fn" >/dev/null || die "Vendor module '$vendor' does not define $fn" |
|
"$fn" "$dest" "$release_id" "$release_dir" |
|
} |
|
|
|
sync_release_to_destination() { |
|
local dest="$1" |
|
local release_dir="$2" |
|
|
|
local vendor tag |
|
vendor="$(get_dest_var "$dest" "VENDOR")" |
|
tag="$(jq -r '.tag_name' "$release_dir/release.json")" |
|
|
|
[[ -n "$vendor" ]] || { |
|
log "ERROR" "Destination '$dest' has no DEST_${dest}_VENDOR" |
|
return 1 |
|
} |
|
|
|
load_vendor_module "$vendor" |
|
|
|
log "INFO" "Destination '$dest': checking release tag '$tag'" |
|
|
|
if destination_release_exists "$dest" "$vendor" "$tag"; then |
|
log "INFO" "Destination '$dest': release '$tag' already exists, skipping" |
|
return 0 |
|
fi |
|
|
|
log "INFO" "Destination '$dest': release '$tag' is missing, creating" |
|
local release_id |
|
release_id="$(destination_create_release "$dest" "$vendor" "$release_dir")" |
|
|
|
[[ -n "$release_id" ]] || { |
|
log "ERROR" "Destination '$dest': vendor returned empty release id" |
|
return 1 |
|
} |
|
|
|
destination_upload_assets "$dest" "$vendor" "$release_id" "$release_dir" |
|
|
|
log "INFO" "Destination '$dest': release '$tag' mirrored" |
|
} |
|
|
|
main() { |
|
parse_args "$@" |
|
|
|
require_command bash |
|
require_command curl |
|
require_command jq |
|
require_command date |
|
require_command mktemp |
|
require_command mkdir |
|
require_command basename |
|
require_command find |
|
require_command sort |
|
require_command sha256sum |
|
|
|
load_config |
|
prepare_workdir |
|
trap cleanup EXIT |
|
|
|
log "INFO" "Release artifact mirror run started" |
|
|
|
local releases_json |
|
releases_json="$(fetch_source_releases)" |
|
|
|
local total |
|
total="$(jq 'length' "$releases_json")" |
|
log "INFO" "GitHub releases fetched: $total" |
|
|
|
local success_count=0 |
|
local failure_count=0 |
|
local skipped_count=0 |
|
|
|
local i release_tmp tag release_dir dest |
|
for (( i=0; i<total; i++ )); do |
|
release_tmp="$WORK_DIR/release-$i.json" |
|
jq ".[$i]" "$releases_json" > "$release_tmp" |
|
|
|
tag="$(jq -r '.tag_name' "$release_tmp")" |
|
[[ -n "$tag" && "$tag" != "null" ]] || die "GitHub release index $i has no tag_name" |
|
|
|
if ! release_is_selected "$release_tmp"; then |
|
log "INFO" "Skipping GitHub release '$tag' due to draft/prerelease filters" |
|
skipped_count=$((skipped_count + 1)) |
|
continue |
|
fi |
|
|
|
release_dir="$(materialize_release "$release_tmp" "$tag")" |
|
|
|
for dest in "${DESTINATIONS[@]}"; do |
|
if sync_release_to_destination "$dest" "$release_dir"; then |
|
success_count=$((success_count + 1)) |
|
else |
|
failure_count=$((failure_count + 1)) |
|
fi |
|
done |
|
done |
|
|
|
log "INFO" "Release artifact mirror run finished: success=$success_count failure=$failure_count skipped=$skipped_count" |
|
|
|
[[ "$failure_count" -eq 0 ]] || exit 2 |
|
} |
|
|
|
main "$@" |