Pipster – update PyPI dependencies

pipster.sh

#!/usr/bin/env bash

# Update the Python packages listed in requirements.top and their dependencies, then "freeze"
# those versions into requirements.txt. If anything has changed, push a branch and open or update
# a pull request.

set -evo pipefail

GITHUB_AUTH_URL="https://$GITHUB_TOKEN:[email protected]/"
WORKDIR="/tmp/pipster-work"
GH="gh.$(arch)"
R=requirements

git config --global user.name "${GIT_NAME-Pipster}"
git config --global user.email "${GIT_EMAIL-pipster@example.com}"
git config --global url.$GITHUB_AUTH_URL.insteadOf https://github.com/

# Create a working dir and clone the repo into it, then check out a branch for our pull request
rm -rf $WORKDIR
git clone --depth 1 ${REPO_URL}.git $WORKDIR
cd $WORKDIR

# Make sure we have a remote for the base branch, then check out a branch for our pull request
set $(git ls-remote --symref origin HEAD)
git fetch origin refs/heads/${BASE_BRANCH=${2##*/}}:refs/remotes/origin/$BASE_BRANCH
git checkout -b "${PR_BRANCH=Pipster-update-dependencies}" "origin/$BASE_BRANCH"

# Install non-Python dependencies if needed
test -f $R.apt && apt-get update && apt-get install -y $(< $R.apt)

# Split requirements.top into PyPI package names and GitHub URLs + packages marked "# pin"
grep -Ev '^(#|$|git\+)|@ +git\+|# pin' $R.top | cut -d= -f 1 | sort -f > $R.unpinned
grep -E '^git\+|@ +git\+|# pin$' $R.top > $R.pinned && true

# Install current versions of our top-level PyPi packages, and the pinned packages,
# and all their dependencies
pip install -U -r $R.unpinned -r $R.pinned

# Save updated package and dependency versions into requirements.txt
pip freeze | sort -ft = -k 1,1 > $R.txt

# Exit the script if all this didn't change requirements.txt
git add $R.txt
git commit --dry-run

# Reassemble requirements.top, by joining the package names we split out earlier with the
# matching lines from the new requirements.txt, then adding the pinned packages back at the end
printf '# Edit requirements here, then run bin/refreeze.sh\n\n' > $R.top

# Make "[extra]" dependencies look like a separate field to `join`
sed -i -e 's/\[/=[/' $R.unpinned

# Match package names from requirements.top with the new versions in requirements.txt
join -it = -o 2.1,1.2,2.3 $R.unpinned $R.txt >> $R.top

# Move the "=" delimiter from before the "[extra]" to after it
sed -i -E 's/=\[([^]]+)\]=/[\1]==/' $R.top

# Add the packages we didn't update back to requirements.top
cat $R.pinned >> $R.top

# If the dry-run env var is set, just print out the requirements files and exit
test $DRY_RUN && tail -n +1 $R.t{op,xt} && exit

# If our previous pull request for this repo is still open, close it.
# (We can't just reuse it, its body has an old diff.)
$GH pr close "$PR_BRANCH" || true

# Commit our changes and push our branch to GitHub
git clean -f
git add $R.top
git commit -m "${TITLE=Pipster: update dependencies and refreeze}"
git push --force origin "$PR_BRANCH":"$PR_BRANCH"

# Create our pull request
PIPSTER_URL="https://github.com/$GITHUB_ORG/pipster"
HEADER="This PR was generated by [Pipster]($PIPSTER_URL) to update these PyPI packages:"
CHANGES=$(git diff origin/$BASE_BRANCH $R.txt | grep '^[+-][^+-]' | sort -k 1.2V)
BODY=$(printf '%s\n```\n%s\n```' "$HEADER" "$CHANGES")
$GH pr create -f --head "$PR_BRANCH" --reviewer "$TEAM" --body "$BODY"