Mirrors repositories with groups while maintaining nesting between two GitLab instances

gitlab_repo_mirror.sh

#!/usr/bin/env bash
set -euo pipefail

# Mirrors repositories with groups while maintaining nesting
# between two GitLab instances
#
# Limited to 100 groups/repositories per request,
# if you need more, you need to update this script!
# As an example of pagination implementation
# https://gist.github.com/WoozyMasta/03c4c532669f43dca0de11087f24d077

: "${SOURCE_GITLAB_HOST:?}"
: "${SOURCE_GITLAB_TOKEN:?}"
: "${SOURCE_GITLAB_GROUP_ID:?}"

: "${TARGET_GITLAB_HOST:?}"
: "${TARGET_GITLAB_TOKEN:?}"
: "${TARGET_GITLAB_GROUP_ID:?"

gitlab() {
  local url="${1:?}" token="${2:?}" path="${3:?}" args=("${@:4}")

  curl -sfH "PRIVATE-TOKEN: $token" \
    --url "https://$url/api/v4/$path" "${args[@]}"
}

gitlab::src() {
  gitlab "$SOURCE_GITLAB_HOST" "$SOURCE_GITLAB_TOKEN" "$1" "${@:2}"
}

gitlab::dst() {
  gitlab "$TARGET_GITLAB_HOST" "$TARGET_GITLAB_TOKEN" "$1" "${@:2}"
}

urlencode() {
  local raw="$*"

  for (( i=0; i<${#raw}; i++ )); do
    local c="${raw:$i:1}"
    case "$c" in
      [a-zA-Z0-9.~_-])   printf '%s' "$c";;
      *)                 printf '%%%02X' "'$c";;
    esac
  done
}

get_group() {
  local parent_id="$1" name="$2" path="$3" vis="$4" desc="$5" group_id

  group_id="$(
    gitlab::dst "groups?search=$path" \
    | jq -er --arg parent "$parent_id" --arg path "$path" '
        .[] |
        select((.parent_id|tostring) == $parent and .path == $path) |
        .id // 0
      ' || echo 0
  )"

  if [ "${#group_id}" -ne 0 ] && [ "$group_id" -ne 0 ]; then
    echo "$group_id"
    return
  fi

  gitlab::dst groups -X POST \
    --data "name=$name" \
    --data "path=$path" \
    --data "parent_id=$parent_id" \
    --data "visibility=$vis" \
    --data "description=$desc" \
  | jq -er '.id // 0' || echo 0
}

get_project() {
  local parent_id=$1 name=$2 path=$3 vis=$4 branch=$5 desc=$6 project_id

  project_id="$(
    gitlab::dst "projects?search=$path" | \
    jq -er --arg ns "$parent_id" --arg path "$path" '
      .[] |
      select((.namespace.id|tostring) == $ns and .path == $path) |
      .id // 0' || echo 0
  )"

  if [ "${#project_id}" -ne 0 ] && [ "$project_id" -ne 0 ]; then
    echo "$project_id"
    return
  fi

  curl -sfX POST -H "PRIVATE-TOKEN: $TARGET_GITLAB_TOKEN" \
    "https://$TARGET_GITLAB_HOST/api/v4/projects" \
    --data "name=$name" \
    --data "path=$path" \
    --data "namespace_id=$parent_id" \
    --data "visibility=$vis" \
    --data "default_branch=$branch" \
    --data "description=$desc" \
    --data "jobs_enabled=false" \
    | jq -er '.id // 0' || echo 0
}

mirror_repo() {
  local source_url="$1" target_url="$2"
  local dir source_auth_url target_auth_url exist=false

  dir="${source_url//*"$SOURCE_GITLAB_HOST/"}"
  source_auth_url="https://oauth2:$SOURCE_GITLAB_TOKEN@${source_url#*//}"
  target_auth_url="https://oauth2:$TARGET_GITLAB_TOKEN@${target_url#*//}"

  if [ -d "$dir" ]; then
    printf '%-10s %s' 'Fetch:' "$source_url"
    git -C "$dir" fetch origin &>/dev/null \
    || printf '\r\e[2K%-10s %s\n' 'Failed:' "$source_url"
    exist=true
  else
    printf '%-10s %s' 'Clone:' "$source_url"
    git clone --mirror "$source_auth_url" "$dir" &>/dev/null \
    || printf '\r\e[2K%-10s %s\n' 'Failed:' "$source_url"
  fi

  if [ "$(git -C "$dir" rev-list --count --all || echo 0)" -eq 0 ]; then
    printf '\r%-10s\n' 'Skip:'
    return
  fi

  git -C "$dir" remote remove target &>/dev/null || :
  git -C "$dir" remote add target "$target_auth_url" &>/dev/null

  if [ "$exist" == true ]; then
    git -C "$dir" config --add remote.target.push '+refs/heads/*:refs/heads/*'
    git -C "$dir" config --add remote.target.push '+refs/tags/*:refs/tags/*'
    git -C "$dir" config --add remote.target.push '+refs/change/*:refs/change/*'
  fi

  if ! git -C "$dir" push --mirror target &>/dev/null; then
    printf '\r\e[2K%-10s %s\n' 'Failed:' "$target_url"
    return
  fi

  if [ "$exist" == true ]; then
    printf '\r\e[2K%-10s %s\n' 'Update:' "$target_url"
  else
    printf '\r\e[2K%-10s %s\n' 'Pushed:' "$target_url"
  fi
}

mirror_group() {
  local src_group_id="$1" dst_group_id="$2"
  if [ "${#src_group_id}" -eq 0 ] && [ "$src_group_id" = '' ]; then
    echo "Empty source group ID"
    return
  fi
  if [ "${#dst_group_id}" -eq 0 ] && [ "$dst_group_id" = '' ]; then
    echo "Empty target group ID"
    return
  fi

  # Source Sub-Group (ssg)
  while IFS=$'\t' read -r ssg_id ssg_name ssg_path ssg_vis ssg_desc; do
    mirror_group "$ssg_id" "$(
      get_group "$dst_group_id" "$ssg_name" \
        "$ssg_path" "$ssg_vis" "${ssg_desc//[$'\r\n']}"
    )"
  done < <(
    gitlab::src "groups/$src_group_id/subgroups?per_page=100" \
    | jq -er '.[] | [.id, .name, .path, .visibility, .description] | @tsv'
  )

  while IFS=$'\t' read -r p_name p_path p_url p_vis p_branch p_desc; do
    p_id="$(
      get_project "$dst_group_id" "$p_name" \
        "$p_path" "$p_vis" "$p_branch" "${p_desc//[$'\r\n']}"
    )"
    if [ "${#p_id}" -eq 0 ] && [ "$p_id" = '' ]; then
      echo "Empty project ID for $p_name"
      continue
    fi

    mirror_repo "$p_url" "$(
      gitlab::dst "projects/$p_id" | jq -er '.http_url_to_repo'
    )"
  done < <(
    gitlab::src "groups/$src_group_id/projects?per_page=100" \
    | jq -er '.[] | [
      .name, .path, .http_url_to_repo,
      .visibility, .default_branch, .description
    ] | @tsv'
  )
}

cd "${0%/*}"

echo "Project mirroring will be started, information:"
echo "  Source GitLab:      $SOURCE_GITLAB_HOST group $SOURCE_GITLAB_GROUP_ID"
echo "  Target GitLab:      $TARGET_GITLAB_HOST group $TARGET_GITLAB_GROUP_ID"
echo "  Working directory:  $PWD"
echo
read -rn1 -p "Press any key to continue"
printf '\r\e[2K%s\n' 'Start fetching groups and repositories...'

mirror_group "$SOURCE_GITLAB_GROUP_ID" "$TARGET_GITLAB_GROUP_ID"
echo "Done."