Skip to content

Instantly share code, notes, and snippets.

@smoser

smoser/README.md

Last active June 30, 2026 14:32
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save smoser/0a11e2643b884960c1e5349d4dc0b8c7 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save smoser/0a11e2643b884960c1e5349d4dc0b8c7 to your computer and use it in GitHub Desktop.
Download ZIP
wolfi get file list and apk info
Raw
README.md

Random wolfi and tools.

  • get-archive-info - get a tar tvf output and the .APKINFO for every file in the archive.

  • build-stage - throw a bunch of files and see which build. they do not depend on each other (each only builds with the wolfi repo)

    I used this to help create batches of things when changing lots of files.

  • test-installable - its like the c-i test that checks that all packages that were built are installable.

  • docker-run-action - it is like the docker run action

  • get-log - get a build or test log for a package.

Raw
build-stage
#!/bin/bash
# shellcheck disable=SC2015,SC2039,SC2166,SC3043
#
# the goal of this script was to ultimately be given a long
# list of packages and to sort out batches of packages that
# depended on another so that they could be submitted. wolfictl's
# dependency resolution / build-order solving was having problems
# and queing up N batches that could land in order was nice.
#
# Other thing that this does well is give nice per-package logs.
VERBOSITY=0
TEMP_D=""
stderr() { echo "$@" 1>&2; }
fail() { local r=$?; [ $r -eq 0 ] && r=1; failrc "$r" "$@"; }
failrc() { local r="$1"; shift; [ $# -eq 0 ] || stderr "$@"; exit "$r"; }
Usage() {
cat <<EOF
Usage: ${0##*/} [ options ] output-dir/ package1 package2 ....
build packages one by one.
each package builds with access to wolfi repo only.
successful builds populate output-dir/packages/
logs for each build are put in output-dir/logs/<package>.log
summary log written to output-dir/build.log
options:
-h | --help show usage
-v | --verbose increase verbosity
-j | --jobs N use N processes
--skip-test do not 'melange test'
--skip-install do not test install of output.
EOF
}
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || stderr "$@"; return 1; }
cleanup() {
[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
debug() {
local level="$1"
shift
[ "${level}" -gt "${VERBOSITY}" ] && return
stderr "${@}"
}
do_build_Usage() {
cat <<EOF
${0##*/} do-build [options] log-dir packages-dir package ...
options:
--summary file write short log to file
EOF
}
sumlog() {
local f="$SUMLOG"
if [ -n "$f" ]; then
echo "$@" >> "$f"
else
echo "$@" 1>&2
fi
}
vrun() {
local rc=0
stderr "running:" "$@"
"$@" || rc=$?
[ $rc -eq 0 ] && stderr "success" || stderr "fail: $rc"
stderr
return $rc
}
dothing() {
local msg="$1" log="$2" rc=0 rs="SUCCESS" opstart="$SECONDS"
shift 2
sumlog "START $msg [${log##*/}]"
vrun "$@" >>"$log" 2>&1 ||
{ rc=$? ; rs=FAIL; }
sumlog "END $msg $rs [$((SECONDS-opstart))s]"
return $rc
}
build_index() {
# FIXME: this should probably lock
local pdir="$1" key="" d=""
shift
[ $# -ge 1 ] && key=$1 && shift
for arch in x86_64 aarch64; do
[ -d "$pdir/$arch" ] || continue
( cd "$pdir/$arch" && set -- *.apk;
[ -f "$1" ] || {
stderr "no .apk files in $PWD ($1 not a file)";
exit 1;
}
melange index ${key:+"--signing-key=$key"} "$@"
) || {
stderr "failed to generate index in $pdir/$arch"
return 1
}
done
return 0
}
publish() {
local src="${1%/}" dest="$2" key="$3"
rsync -a --include="*.apk" --include='*/' \
--exclude='*' "$src/" "$dest"
build_index "$dest" ${key:+"$key"} ||
fail "failed to build index in $dest"
}
# shellcheck disable=SC2115
empty_dir() {
local d="$1"
[ "$d" = "/" -o "$d" = "" ] &&
{ stderr "empty_dir '$d' - not doing that"; return 1; }
[ -d "$d" ] || return 0
(
# all gets done in a subprocess do i don't have to change dir back.
shopt -s nullglob
cd "$d" || { stderr "could not cd '$d'"; exit 1; }
set -- * .?*
[ $# -eq 0 ] && exit 0
lf=$(mktemp) || exit 1
trap 'rm -Rf "$lf"' EXIT
# don't want to spew errors
rm -Rf "$@" >"$lf" 2>&1 && exit 0
rrc0=$? crc=0 rrc1=0
chmod -R u+w "$@" >> "$lf" 2>&1
crc=$?
if rm -Rf "$@" >"$lf" 2>&1; then
stderr "empty_dir($d) required chmod u+w (rrc0=$rrc0 crc=$crc)"
exit 0
fi
rrc1=$?
stderr "empty_dir($d) failed $rrc1 (rrc0=$rrc0 crc=$crc)"
tail -n 10 "$lf" 1>&2
exit $rrc1
)
}
do_build() {
local short_opts="hv"
local long_opts="help,skip-build,skip-test,skip-install,summary:,verbose"
local getopt_out=""
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
{ bad_Usage; return; }
local summary="" dbuild=true dtest=true dinstall=true
while [ $# -ne 0 ]; do
{ cur="$1"; next="$2"; }
case "$cur" in
-h|--help) Usage ; exit 0;;
--summary) summary="$next"; shift;;
--skip-build) dbuild=false;;
--skip-test) dtest=false;;
--skip-install) dinstall=false;;
-v|--verbose) VERBOSITY=$((VERBOSITY+1));;
--) shift; break;;
esac
shift;
done
local logd="$1" trepo="$2"
shift 2
SUMLOG="$summary"
mkdir -p "$logd" "$trepo" || fail "failed making dirs"
TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/build-XXXXXX") ||
fail "failed to make tempdir"
trap cleanup EXIT
local lkey=""
[ -f "local-melange.rsa" ] && lkey="$PWD/local-melange.rsa"
local merepos="" ierepos="" r="" stages=""
# a 'stage' here would be the output of another build-stage
# where we include it's packages for use.
# shellcheck disable=SC2206
stages=( ${MY_STAGES} )
for r in "${stages[@]}"; do
merepos="$merepos --repository-append=$r/packages"
ierepos="$ierepos --repo=$r/packages"
done
merepos=${merepos# }
ierepos=${ierepos# }
local pkg="" pstart="" log="" rs="" rc="" pfail=""
for pkg in "$@"; do
pstart=$SECONDS
pkg=${pkg%.yaml}
log="$logd/$pkg.log"
: > "$log" || fail "failed writing to $log"
echo " ${#stages[@]} stages - ${stages[*]}" >> "$log"
echo "start $pkg $log"
pdir="$TEMP_D/$pkg"
prepo="$pdir/repo"
tdir="$TEMP_D/$pkg.tmp"
mkdir -p "$prepo" "$tdir" || fail "failed create prepo dir"
pfail=""
if [ $dbuild = true ]; then
dothing "$pkg build" "$log" \
env TMPDIR="$tdir" \
make "package/$pkg" "REPO=$prepo" \
MELANGE_EXTRA_OPTS="--out-dir=$prepo $merepos" ||
pfail=build
empty_dir "$tdir"
fi
if [ -z "$pfail" ] && [ $dtest = true ]; then
dothing "$pkg test" "$log" \
env TMPDIR="$tdir" \
make "test/$pkg" "REPO=$prepo" \
"MELANGE_EXTRA_OPTS=$merepos" ||
pfail="test"
empty_dir "$tdir"
fi
if [ -z "$pfail" ] && [ $dinstall = true ]; then
# shellcheck disable=SC2086
dothing "$pkg test-install" "$log" \
test-installable "--repo=$prepo" $ierepos ||
pfail="install"
empty_dir "$tdir"
fi
if [ -z "$pfail" ]; then
## FIXME: need a local signing key here. just using local-signing
dothing "$pkg publish" "$log" \
publish "$prepo" "$trepo" ${lkey:+"$lkey"} ||
pfail="publish"
fi
if [ -n "$pfail" ]; then
rs="FAIL/$pfail"
fails="$fails $pkg"
else
rs="PASS"
fi
rm -Rf "$pdir"
echo "finish $pkg $rs $log [$((SECONDS-pstart))s]"
sumlog "RESULT $pkg $rs [$((SECONDS-pstart))s] ${log##*/}"
done
if [ -n "$fails" ]; then
return 1
fi
return 0
}
main() {
local short_opts="hj:v"
local long_opts="help,jobs:,skip-build,skip-test,skip-install,verbose"
local getopt_out=""
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
{ bad_Usage; return; }
local cur="" next="" pt="" jobs=1
pt=( )
while [ $# -ne 0 ]; do
# shellcheck disable=SC2034
{ cur="$1"; next="$2"; }
case "$cur" in
-h|--help) Usage ; exit 0;;
-j|--jobs) jobs=$next; shift;;
-v|--verbose) VERBOSITY=$((VERBOSITY+1));;
--skip-build|--skip-test|--skip-install) pt[${#pt[@]}]="$cur";;
--) shift; break;;
esac
shift;
done
mkdir -p ~/.parallel;
: > ~/.parallel/will-cite
[ $# -ge 2 ] || {
bad_Usage "must provide arguments. got $# ($*) expect 2+";
return;
}
local outd="$1" outd_in="$1"
shift
mkdir -p "$outd_in" || fail "failed create out-dir '$outd_in'"
outd=$(cd "$outd_in" && pwd) || fail "failed to get real path to $outd_in"
mkdir -p "$outd/packages" "$outd/logs" ||
fail "failed mkdir $outd/packages $outd/logs"
TEMP_D=$(mktemp -d "$outd/.build-XXXXXX") ||
fail "failed to make tempdir"
trap cleanup EXIT
local workd="$TEMP_D"
local pkg="" sumlog="$outd/logs/short.log" fails=""
local failn=0 passn=0
: > "$sumlog"
echo "building $# packages in $jobs jobs. output in $outd."
TMPDIR="$workd"
parallel \
"--jobs=$jobs" --line-buffer -- \
"$0" do-build "${pt[@]}" "--summary=$sumlog" \
"$outd/logs" "$outd/packages" \
::: "$@"
rc=$?
awk '$1 == "RESULT" && $3 ~ "^FAIL" { print $2 }' "$sumlog" > "$outd/logs/fails"
awk '$1 == "RESULT" && $3 ~ "PASS" { print $2 }' "$sumlog" > "$outd/logs/passes"
sort "$outd/logs/fails" "$outd/logs/passes" > "$outd/logs/attempts"
failn=$(wc -l < "$outd/logs/fails")
passn=$(wc -l < "$outd/logs/passes")
echo "$((failn+passn)) attempts. $failn fails. $passn pass. took $((SECONDS))s"
[ "$failn" -eq 0 ]
}
if [ "$1" = "do-build" ]; then
shift
do_build "$@"
exit
elif [ "$1" = "publish" ]; then
shift
publish "$@"
exit
fi
main "$@"
# vi: ts=4 expandtab
Raw
build-stages
#!/bin/bash
#
# needs usage and stuff, but just commit it for now.
#
# Basically this calls 'build-stage' until it doesn't
# accomplish anything. so you will end up with 'stageXX'
# directories each having built more things.
#
# When I did this with the 184 ruby 3.2 files changing to
# ruby 3.3, it took 6 stages. All packages are built in
# isolation in a stage, only given access to previous
# stages and archive.
set -o pipefail
Usage() {
cat <<EOF
Usage: ${0##*/} out-base-dir package [...]
build provided packages in stages
out-base-dir will get logs and stage builds.
EOF
}
vrun() {
local out="$1"
local start=$SECONDS
shift
{
echo "execute: $*"
time "$@"
r=$?
echo "r=$r [$((SECONDS-start))s]"
} 2>&1 | tee "$out"
}
check_output() {
local rc="$1" stageoutd="$2" missing=""
for f in logs/fails logs/passes ; do
[ -e "$stageoutd/$f" ] || {
echo "$f in $stageoutd/ did not exist" 1>&2
missing="$missing $f"
}
[ -f "$stageoutd/$f" ] || {
echo "$f in $stageoutd/ was not a file"
missing="${missing} $f"
continue
}
[ -r "$stageoutd/$f" ] || {
missing="${missing} $f"
echo "$f in $stageoutd/ was not readable"
continue
}
done
if [ -z "$missing" ]; then
return 0
fi
echo "rc=$rc stage output dir $stageoutd missing ${missing# }"
return "$rc"
}
if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
Usage
exit 0
fi
[ $# -gt 1 ] || {
Usage 1>&2
echo "got $# , need more than 1"
exit 1
}
outd="$1"
shift
[ -d "$outd" ] || mkdir -p "$outd" ||
{ echo "failed to create $outd"; exit 1; }
outd=$(cd "$outd" && pwd) ||
{ echo "failed to get full path to $outd"; exit 1; }
njobs=16
n=1
max=20
n=0
to_build="$*"
to_buildn=$#
lto_buildn=0
lstage=""
MY_STAGES=""
while n=$((n+1)) && [ $n -lt $max ]; do
if [ -n "$lstage" ]; then
to_build=$(cat "$outd/$lstage/logs/fails")
to_buildn=$(wc -l < "$outd/$lstage/logs/fails")
fi
if [ "$to_buildn" -eq 0 ]; then
echo "stage '$lstage' had no failures"
exit 0
fi
if [ "$to_buildn" -eq "$lto_buildn" ]; then
echo "Quitting, stage $lstage did not make progress ($to_buildn fails)"
exit 1
fi
stage=$(printf "stage%02d" "$n")
echo "===== stage $stage to_buildn=$to_buildn (lstage=$lstage) ====="
MY_STAGES="${MY_STAGES:+${MY_STAGES} }$outd/$lstage"
# shellcheck disable=SC2086
set -- $to_build
vrun "$outd/log-$stage.txt" \
env MY_STAGES="$MY_STAGES" \
build-stage "-j$njobs" "$outd/$stage" "$@"
check_output "$?" "$outd/$stage" || {
echo "stage $stage failed without producing fails/passes in $outd/$stage"
exit 1
}
lstage=$stage
lto_buildn=$to_buildn
fails=$(wc -l < "$outd/$stage/logs/fails")
passes=$(wc -l < "$outd/$stage/logs/passes")
echo "==== stage $stage attempts=$to_buildn fails=$fails passes=$passes ==="
done
echo "Quitting n=$n"
Raw
create-multiple-prs
#!/bin/sh
# shellcheck disable=SC3043,SC2059,SC2015,SC2162
#
# this is here as demonstration of 'gh' usage and cutting up a mega-pr
# into smaller prs.
# I had made https://github.com/wolfi-dev/os/pull/45833 which changed 555
# packages one commit each (using a python program) and then
# needed those split up to get them landed.
#
# To do so:
# git log --no-decorate --no-abbrev-commit --format=oneline \
# upstream/main..test/tw/ldd-check-the-things > list.txt
# create-multiple-prs < list.txt
fail() { echo "$@" 1>&2; exit 1; }
vr() { echo "$ $*" 1>&2; "$@"; return; }
finish() {
local branch="$1" num="$2" numpad="" githubname="smoser" out=""
cat >pr-body.txt <<EOF
This cleans up use of ldd-check, replacing
test/ldd-check with test/tw/ldd-check and dropping
the defaults where applicable.
EOF
numpad=$(printf "%02d" "$num")
echo "finish $branch [$num]"
vr git push $githubname HEAD || fail "failed push $githubname HEAD"
out=$(vr gh pr create \
--label="automated pr" \
--label="smoser/tw-ldd-check" \
--base=wolfi-dev/os/main --body-file=pr-body.txt \
--title="tw/ldd-check cleanup batch $numpad" \
--repo=wolfi-dev/os --base=main \
"--head=$githubname:$branch" </dev/null) || fail "bad - $branch/$num"
echo "$branch - $out" | tee -a prs.txt
#gh pr create --label="automated pr" --draft --body-file=pr-body.txt --title="tw/ldd-check cleanup batch 1" --base=main --repo=wolfi-dev/os --head=smoser:tw/ldd-check-cleanup-01
}
cutup() {
local branchfmt="tw/ldd-check-cleanup-%02d"
local branchnum=1 batchsize=15
while read commit _desc; do
fullnum=$((fullnum+1))
if [ $((fullnum%batchsize)) -eq 1 ]; then
branchname=$(printf "$branchfmt" "$(((fullnum/batchsize)+1))")
vr git checkout -b "$branchname" upstream/main &&
vr git reset --hard upstream/main ||
fail "failed branch create for $branchname"
fi
vr git cherry-pick "$commit" || fail "cherry-pick $commit failed"
if [ $((fullnum%batchsize)) -eq 0 ]; then
finish "$branchname" "$((fullnum/batchsize))" || fail
branchnum=$((branchnum+1))
[ $branchnum -gt 40 ] && { echo "skipping out"; exit 0; }
fi
done
if [ "$fullnum" -eq 0 ]; then
fail "didn't get any input?"
fi
if [ $((fullnum%batchsize)) -ne 0 ]; then
finish "$branchname" $((fullnum/batchsize)) || fail
fi
}
# run this with input of 'git log upstream/main..HEAD
# git log --no-decorate --no-abbrev-commit --format=oneline upstream/main..HEAD | go-cut.sh
cutup
Raw
docker-run-action
#!/bin/bash
# shellcheck disable=SC2015,SC2039,SC2166,SC3043
IMAGE="ghcr.io/wolfi-dev/sdk:latest"
VERBOSITY=0
TEMP_D=""
stderr() { echo "$@" 1>&2; }
fail() { local r=$?; [ $r -eq 0 ] && r=1; failrc "$r" "$@"; }
failrc() { local r="$1"; shift; [ $# -eq 0 ] || stderr "$@"; exit "$r"; }
Usage() {
cat <<EOF
Usage: ${0##*/} [ options ] command args
Taken from .github/actions/docker-run/action.yaml
Run stuff in a container.
options:
-h | --help help
-v | --verbose increase verbosity
-i | --image IMAGE image to run [default=$IMAGE]
-w | --workdir DIR default is current working dir
-e | --entrypoint entrypoint - default is /bin/bash
-t | --tty use a tty - default is to give tty if /dev/stdin is tty
--no-tty do not use --tty
--pt OPT pass through to docker run
Example:
${0##*/} --repo=packages packages/*.apk
EOF
}
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || stderr "$@"; return 1; }
cleanup() {
[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
debug() {
local level="$1"
shift
[ "${level}" -gt "${VERBOSITY}" ] && return
stderr "${@}"
}
main() {
local short_opts="hi:w:e:tr:v"
local long_opts="help,entrypoint:,image:,no-tty,no-priv,tty,pt:,volume:,workdir:,verbose"
local getopt_out=""
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
{ bad_Usage; return; }
local cur="" next="" pt="" priv=true
local tty="" image="$IMAGE" workdir="$PWD" entrypoint="/bin/bash"
pt=( )
while [ $# -ne 0 ]; do
{ cur="$1"; next="$2"; }
case "$cur" in
-h|--help) Usage ; exit 0;;
-i|--image) image=$next; shift;;
-w|--workdir) workdir=$next; shift;;
-e|--entrypoint) entrypoint=$next; shift;;
-t|--tty) tty=true;;
--no-tty) tty=false;;
--pt) pt[${#pt[@]}]="$next"; shift;;
-r|--repo) repo="$next"; shift;;
--volume) volumes[${#volumes[@]}]="--volume=$next"; shift;;
-v|--verbose) VERBOSITY=$((VERBOSITY+1));;
--no-priv) priv=false;;
--) shift; break;;
esac
shift;
done
local privops="" ttyflag=""
if [ -z "$tty" ] ; then
[ -t 0 ] && tty="true" || tty=false
fi
[ $tty = "true" ] && ttyflag="--tty"
local home="${HOME:-/home/user}"
case "$image" in
*.*/*) :;;
*) image="cgr.dev/chainguard-private/$image";;
esac
case "$entrypoint" in
none) entrypoint="";;
esac
privops=(
--privileged
--security-opt="seccomp=unconfined"
--security-opt="apparmor:unconfined" \
)
[ "$priv" = "true" ] || privops=( )
docker run \
"${privops[@]}" \
--volume="$workdir:$workdir" \
--volume="$home:$home" \
"${volumes[@]}" \
--workdir="$workdir" \
${entrypoint:+"--entrypoint=${entrypoint}"} \
--env "HOME=$home" \
--interactive ${ttyflag:+"${ttyflag}"} \
"${pt[@]}" \
"$image" "$@"
}
main "$@"
# vi: ts=4 expandtab
Raw
get-archive-info
#!/bin/bash
# shellcheck disable=SC2015,SC2039,SC2166,SC3043
#
# the goal of this script was to ultimately be given a long
# list of packages and to sort out batches of packages that
# depended on another so that they could be submitted. wolfictl's
# dependency resolution / build-order solving was having problems
# and queing up N batches that could land in order was nice.
#
# Other thing that this does well is give nice per-package logs.
VERBOSITY=0
TEMP_D=""
WOLFI_ARCHIVE="https://packages.wolfi.dev/os"
WOLFI_ARCHIVE="https://apk.cgr.dev/chainguard"
ENTERPRISE_ARCHIVE="https://apk.cgr.dev/chainguard-private"
EXTRAS_ARCHIVE="https://packages.cgr.dev/extras"
DEF_ARCHIVE="$WOLFI_ARCHIVE"
stderr() { echo "$@" 1>&2; }
fail() { local r=$?; [ $r -eq 0 ] && r=1; failrc "$r" "$@"; }
failrc() { local r="$1"; shift; [ $# -eq 0 ] || stderr "$@"; exit "$r"; }
Usage() {
cat <<EOF
Usage: ${0##*/} [ options ] output-dir/ [package1 package2] ....
Grab information about packages in archive.
options:
-h | --help show usage
-v | --verbose increase verbosity
-j | --jobs N use N processes - default to 1 per cpu
-a | --archive A use archive rather than default (wolfi)
--arch ARCH use arch rather than current arch
EOF
}
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || stderr "$@"; return 1; }
cleanup() {
[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
debug() {
local level="$1"
shift
[ "${level}" -gt "${VERBOSITY}" ] && return
stderr "${@}"
}
apk_info_Usage() {
cat <<EOF
${0##*/} apk-info [options] url output-base
EOF
}
# shellcheck disable=SC2120
get_arch() {
local archin="$1" unamem=""
[ -n "$archin" ] && { echo "$archin"; return 0; }
unamem=$(uname -m) || { stderr "failed to get 'uname -m'"; return 1; }
case "$unamem" in
x86_64|aarch64) : ;;
*) stderr "do not know what to do with uname -m output '$unamem'"
return 1;;
esac
echo "$unamem"
}
dl() {
local url="$1" dest="$2" out="" rc=""
[ $# -lt 2 ] && { stderr "dl got $# args ($*) needed 2"; return 1; }
shift 2
set -- ${HTTP_AUTH:+--user "user:${HTTP_AUTH}"} --fail --location "$@"
if [ -z "$dest" -o "$dest" = "-" ]; then
curl --quiet "$@" "$url"
return
fi
out=$(curl "$@" --output "$dest" "$url" 2>&1) &&
return 0
rc=$?
echo "$out" 1>&2
echo "failed $rc dl $url to $dest" 1>&2
return $rc
}
apk_info() {
local short_opts="hv"
local long_opts="help,verbose"
local getopt_out=""
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
{ bad_Usage; return; }
local cur="" next=""
local outbase="" url="" keep_apk=false
while [ $# -ne 0 ]; do
{ cur="$1"; next="$2"; }
case "$cur" in
-h|--help) Usage ; exit 0;;
-v|--verbose) VERBOSITY=$((VERBOSITY+1));;
--) shift; break;;
esac
shift;
done
local url="$1" outbase="$2" tmpd="" out=""
shift 2
TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/apk-info-XXXXXX") ||
fail "failed to make tempdir"
trap cleanup EXIT
tmpd="${TEMP_D}"
bn=${url##*/}
case "$url" in
http*) dl "$url" "$tmpd/$bn" || return;;
*) cp "$url" "$tmpd/$bn" || fail "failed copy $url";;
esac
tar --warning=none --to-stdout -xf "$tmpd/$bn" --occurrence=1 .PKGINFO > "$tmpd/info" &&
mv "$tmpd/info" "${outbase}info" ||
fail "failed reading .PKGINFO"
tar --warning=none -tvf "$tmpd/$bn" > "$tmpd/list" &&
mv "$tmpd/list" "${outbase}flist" ||
fail "$url: could not get list"
if [ "$keep_apk" = "true" ]; then
mv "$tmpd/$bn" "${outbase}apk" ||
fail "$url: failed renaming apk"
fi
echo "${outbase##*/} ${SECONDS}s"
return 0
}
main() {
local short_opts="a:hj:v"
local long_opts="arch:,archive:,help,jobs:,verbose"
local getopt_out=""
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
{ bad_Usage; return; }
local cur="" next="" pt="" jobs=0 archives="" arch=""
pt=( )
archives=( )
while [ $# -ne 0 ]; do
# shellcheck disable=SC2034
{ cur="$1"; next="$2"; }
case "$cur" in
-h|--help) Usage ; exit 0;;
-j|--jobs) jobs=$next; shift;;
-v|--verbose) VERBOSITY=$((VERBOSITY+1));;
-a|--archive)
case "$next" in
enterprise) archives[${#archives[@]}]="$ENTERPRISE_ARCHIVE" ;;
extras) archives[${#archives[@]}]="$EXTRAS_ARCHIVE";;
wolfi) archives[${#archives[@]}]="$WOLFI_ARCHIVE";;
*) archives[${#archives[@]}]="$next";;
esac
shift;;
--arch) arch=$next; shift;;
--) shift; break;;
esac
shift;
done
mkdir -p ~/.parallel;
: > ~/.parallel/will-cite
if [ "$jobs" = "0" ]; then
local out=""
[ -f /proc/cpuinfo ] &&
out=$(grep -c processor /proc/cpuinfo) && jobs=$((out*2))
fi
[ $# -ge 1 ] || {
bad_Usage "must provide arguments. got $# ($*) expect 1+";
return;
}
local outd="$1" outd_in="$1" packages="" archive=""
shift
packages=( "$@" )
mkdir -p "$outd_in" || fail "failed create out-dir '$outd_in'"
outd=$(cd "$outd_in" && pwd) || fail "failed to get real path to $outd_in"
if [ "${#archives[@]}" = "0" ]; then
archives=( "$DEF_ARCHIVE" )
elif [ "${#archives[@]}" -gt 1 ]; then
# TODO : more than one archive
fail "cannot have more than one archive, sorry"
fi
archive=${archives[0]}
archive=${archive%/}
if [ "$archive" = "$ENTERPRISE_ARCHIVE" ]; then
if [ -z "$HTTP_AUTH" ]; then
HTTP_AUTH=$(chainctl auth token --audience=apk.cgr.dev) ||
{ stderr "failed to get token for apk.cgr.dev with chainctl"; return 1; }
fi
fi
if [ -z "$arch" ]; then
arch=$(get_arch) || fail "failed to get arch"
fi
TEMP_D=$(mktemp -d "$outd/.build-XXXXXX") ||
fail "failed to make tempdir"
trap cleanup EXIT
local workd="$TEMP_D"
local jsonl="$workd/archive.json" fullflat="$workd/archive.flat" flat=""
dl "$archive/$arch/APKINDEX.tar.gz" "$workd/apkindex.tar.gz" || {
stderr "failed to get index for $archive/$arch"
return 1
}
apkrane ls --latest --json "$workd/apkindex.tar.gz" > "$jsonl" || {
stderr "failed apkrane ls --latest --json $archive/$arch/APKINDEX.tar.gz";
return 1;
}
# get space delimited origin, version, name, size, filename
local rmold=true jqr=""
jqr='"\(.Origin) \(.Version) \(.Name) \(.Size) \(.Name)-\(.Version).apk"'
jq -r "$jqr" "$jsonl" > "$fullflat.raw"
sort <"$fullflat.raw" > "$fullflat"
local numorigins="" origins="$workd/origins.list"
local fullcurflat="$workd/archive-cur.flat"
## there will be only one package that has the same name as its origin
## so we assume that that package is latest.
## that way we can filter binaries with other versions
awk '$1 == $3 { printf("%s %s\n", $1, $2); }' "$fullflat" | sort > "$origins"
numorigins=$(wc -l < "$origins")
# fullcurflat gets only those packages that have the same version as the origin
grep --fixed-strings --file="$origins" "$fullflat" > "$fullcurflat"
if [ ${#packages[@]} -eq 0 ]; then
flat="$fullcurflat"
rmold=true
else
rmold=false
flat="$workd/packages.flat"
{
for p in "${packages[@]}"; do
awk '$1 == n { print $0 }' "n=$p" "$fullcurflat"
done
} > "$flat"
fi
local numbins="" dlsize="" pinput="$workd/p-input"
numbins=$(wc -l < "$flat")
# remove any .flist/.info files that are not in fullflat
local archivebins="$workd/archive-bins.list"
local dirbins="$workd/dir-bins.list"
local oldbins="$workd/old-bins.list"
local newbins="$workd/new-bins.list"
local existbins="$workd/existing-bins.list"
local newfull="$workd/new-full.flat"
awk '{ printf("%s-%s.apk\n", $3, $2); }' "$flat" > "$archivebins"
( cd "$outd" &&
find . -maxdepth 1 -name "*.flist" -type f |
sed -e 's,^[.]/,,' -e 's,.flist$,,' -e 's,$,.apk,') > "$dirbins"
sort "$archivebins" "$archivebins" "$dirbins" | uniq -u > "$oldbins"
sort "$archivebins" "$dirbins" "$dirbins" | uniq -u > "$newbins"
sort "$archivebins" "$dirbins" | uniq -d > "$existbins"
existed=$(wc -l < "$existbins")
grep --fixed-strings --file="$newbins" "$flat" > "$newfull"
dlsize=$(awk '{sum+=$4}; END {printf("%d\n", sum/1024/1024)}' "$newfull")
awk '{base=$1; sub("[.]apk$","",base); printf("%s/%s\n%s/%s.\n", burl, $1, outd, base) }' \
burl="$archive/$arch" "outd=$outd" "$newbins" >"$pinput" ||
fail "failed to read '$newbins' into '$pinput'"
numnew=$(wc -l <"$newbins")
numold=$(wc -l <"$oldbins")
echo "$numorigins total origins. $numnew (${dlsize}MiB) new APKs." \
"$existed existing APK. $jobs jobs"
# get output-base url
env TMPDIR="$workd" ${HTTP_AUTH:+HTTP_AUTH="${HTTP_AUTH}"} \
parallel \
"--jobs=$jobs" --line-buffer --max-replace-args=2 -- \
"$0" apk-info "${pt[@]}" < "$pinput"
rc=$?
if [ "$rmold" = "true" ] && [ "$numold" -ge 1 ]; then
echo "cleaning local data for $numold local binaries not in archive"
( cd "$outd" && while read bin ; do rm -f "${bin%.apk}".*; done ) < "$oldbins"
fi
echo "finished $numorigins origins, $numbins apks" \
"($existed existing, $numnew new [${dlsize}MiB]) in ${SECONDS}s. Exiting $rc"
exit $rc
}
if [ "$1" = "apk-info" ]; then
shift
apk_info "$@"
exit
fi
main "$@"
# vi: ts=4 expandtab
Raw
get-archive-summary
#!/bin/sh
# shellcheck disable=SC2015,SC2039,SC2166,SC3043,SC2059
archives="
wolfi:https://apk.cgr.dev/chainguard
enterprise:https://apk.cgr.dev/chainguard-private
extras:https://packages.cgr.dev/extras
"
stderr(){ echo "$@" 1>&2; }
fail() { echo "$@" 1>&2; exit 1; }
dl() {
local url="$1" dest="$2" out="" rc=""
[ $# -lt 2 ] && { stderr "dl got $# args ($*) needed 2"; return 1; }
shift 2
set -- ${HTTP_AUTH:+--user "user:${HTTP_AUTH}"} --fail --location "$@"
if [ -z "$dest" -o "$dest" = "-" ]; then
curl --quiet "$@" "$url"
return
fi
out=$(curl "$@" --output "$dest" "$url" 2>&1) &&
return 0
rc=$?
echo "$out" 1>&2
echo "failed $rc dl $url to $dest" 1>&2
return $rc
}
json2txt() { jq -r '"\(.Origin) \(.Version) \(.Name) \(.Size) \(.InstalledSize)"' ; }
info() {
local name="$1" arch="$2" t="$3" file="$4"
local origins="" bins="" installed="" size=""
local fmt="" mft=""
origins=$(awk '{printf("%s\n", $1)}' "$file" | sort -u | wc -l)
bins=$(wc -l < "$file")
installed=$(awk '{sum+=$5}; END { printf("%d\n", sum/1024/1024/1024); }' "$file")
size=$(awk '{sum+=$4}; END { printf("%d\n", sum/1024/1024/1024); }' "$file")
local fmt="%-10s %-7s %-6s %8d %8d %8d %8d\n"
if [ "${HEADER:-1}" = "1" ]; then
HEADER=0
mft=$(echo "$fmt" | tr 'd' 's')
printf "$mft\n" "name" "arch" "type" "origins" bins "dl(GB)" "inst(GB)"
fi
printf "$fmt" "$name" "$arch" "$t" "$origins" "$bins" "$size" "$installed"
}
for pair in $archives; do
name=${pair%%:*}
url=${pair#*:}
HTTP_AUTH=""
if [ "$name" = "enterprise" ]; then
if [ -z "$HTTP_AUTH" ]; then
HTTP_AUTH=$(chainctl auth token --audience=apk.cgr.dev) ||
fail "failed to get token for apk.cgr.dev with chainctl"
fi
fi
for arch in x86_64 aarch64; do
index=$name-$arch.tar.gz
if [ ! -f "$index" ]; then
dl "$url/$arch/APKINDEX.tar.gz" "$index" ||
fail "failed download of $url/$arch/APKINDEX.tar.gz"
fi
apkrane ls --json "$index" > "$name-$arch-all.json" 2>/dev/null &&
apkrane ls --json --latest "$index" > "$name-$arch-latest.json" 2>/dev/null ||
fail "failed apkrane ls $index"
json2txt < "$name-$arch-all.json" > "$name-$arch-all.txt" &&
json2txt < "$name-$arch-latest.json" > "$name-$arch-latest.txt"
info "$name" "$arch" "all" "$name-$arch-all.txt"
info "$name" "$arch" "latest" "$name-$arch-latest.txt"
done
done
Raw
get-log
#!/bin/bash
# shellcheck disable=SC2015,SC2039,SC2166,SC3043
VERBOSITY=0
TEMP_D=""
GL_REPO=${GL_REPO:-wolfi}
GL_ARCH=${GL_ARCH:-amd64}
stderr() { echo "$@" 1>&2; }
fail() { local r=$?; [ $r -eq 0 ] && r=1; failrc "$r" "$@"; }
failrc() { local r="$1"; shift; [ $# -eq 0 ] || stderr "$@"; exit "$r"; }
Usage() {
local me="${0##*/}"
cat <<EOF
Usage: $me [ options ] log-type pkg[/ver] [ver]
get the build or test log for provided pkg and version
log-type is one of 'build' or 'test'
ver is required, but can be provided as part of pkg input as 'pkg/ver'
options:
-v | --verbose increase verbosity
-t | --no-timestamp do not prefix lines with timestamp
-d | --descending output in reverse chronological order (newest first)
-a | --arch ARCH get arch ARCH (default amd64)
'amd64', 'x86_64', 'arm64' or 'aarch64'
default can be set via env GL_ARCH ($GL_ARCH)
-r | --repo REPO wolfi (w), enterprise (e), extras (x)
default can be set via env GL_REPO ($GL_REPO)
shorthand in parenthesis
-l | --limit X limit query to X lines
Examples:
* $me --arch=arm64 build flux-2.7 2.7.5-r1
get arm64 build log for flux-2.7 at 2.7.5-r1
* $me --repo=enterprise build eks-distro-1.32/1.32.29-r1
get build log from enterprise for arch=$GL_ARCH
you can provide pkg and version as 'pkg/ver'
* $me test -t php-8.1-xdebug 3.5.0-r0
get test log from wolfi for php-8.1-xdebug at version 3.5.0-r0
without timestamps
* $me -rx -aamd64 t chainctl/0.2.182-r0
amd64 test log for extras package chainctl
* $me --descending --limit 100 build grep ""
get the most recent 100 lines of a grep build,
version is empty.
EOF
}
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || stderr "$@"; return 1; }
cleanup() {
[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
debug() {
local level="$1"
shift
[ "${level}" -gt "${VERBOSITY}" ] && return
stderr "${@}"
}
main() {
local short_opts="a:Dhl:r:tv"
local long_opts="arch:,descending,help,limit:,repo,timestamp,verbose"
local getopt_out=""
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
{ bad_Usage; return; }
local arch=${GL_ARCH} repo=${GL_REPO}
local cur="" next="" timestamp=true pt="" tac=true
pt=( )
while [ $# -ne 0 ]; do
# shellcheck disable=SC2034
{ cur="$1"; next="$2"; }
case "$cur" in
-h|--help) Usage ; exit 0;;
-v|--verbose) VERBOSITY=$((VERBOSITY+1));;
-a|--arch) arch="$next"; shift;;
-l|--limit) pt+=("--limit=$next"); shift;;
-t|--no-timestamp) timestamp=false;;
-r|--repo) repo="$next"; shift;;
-D|--descending) tac=false;;
--) shift; break;;
esac
shift;
done
## check arguments here
## how many args do you expect?
local ltype="" pkg="" ver="" nargs="$#"
[ $# -ne 0 ] || { bad_Usage "must provide arguments"; return; }
ltype="$1"
shift || bad_Usage "got $nargs arguments, need more"
case "$1" in
*/*) pkg=${1%/*}; ver=${1#*/}; shift;;
*) [ $# -lt 2 ] && { bad_Usage "got $nargs args"; return; }
pkg="$1"
ver="$2"
shift 2;;
esac
case "$ltype" in
b|build) ltype="build";;
t|test) ltype="test";;
*) bad_Usage "log-type must be 'build' or 'test', not $ltype";
return 1;;
esac
case "$arch" in
x86_64|amd64|x) arch=amd64;;
aarch64|arm64|a) arch=arm64;;
*) bad_Usage "bad arch $arch. should be arm64 or amd64"; return;;
esac
case "$repo" in
w|wolfi)
repo=wolfi
project="prod-wolfi-os"
;;
e|enterprise)
repo=enterprise
project="prod-images-c6e5"
;;
x|extras)
repo=extras
project="prod-images-c6e5"
;;
*) bad_Usage "repo must be wolfi (w), enterprise (e), extras (x)"
return;;
esac
TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
fail "failed to make tempdir"
trap cleanup EXIT
local mns="labels.k8s-pod/melange_chainguard_dev/"
local filter="" filters="" cmd="" testbool=""
[ "$ltype" = "build" ] && testbool=false || testbool=true
filters=(
"${mns}arch=$arch"
"${mns}package=$pkg"
${ver:+"${mns}version=$ver"}
"${mns}test=$testbool"
"textPayload:*"
)
for f in "${filters[@]}"; do
filter="${filter}AND $f "
done
filter=${filter#AND }
local outfmt="value(timestamp,textPayload)"
if [ "$timestamp" = "false" ]; then
outfmt="value(textPayload)"
fi
cmd=(
gcloud logging read
--project="$project"
--format="$outfmt"
"${pt[@]}"
"$filter"
)
local start=$SECONDS end=""
debug 1 "execute: ${cmd[*]}"
start=$SECONDS
if [ "$tac" = "true" ]; then
"${cmd[@]}" > "${TEMP_D}/log.out" ||
fail "failed gcloud cmd: ${cmd[*]}"
tac "${TEMP_D}/log.out"
else
"${cmd[@]}" || fail "failed gcloud cmd: ${cmd[*]}"
fi
end=$SECONDS
debug 1 "gcloud logging read took $((end-start))s"
}
main "$@"
# vi: ts=4 expandtab
Raw
get-wheel
#!/bin/sh
# shellcheck disable=SC2015,SC2039,SC2166,SC3043
Usage() {
cat <<EOF
${0##*/} [-e/-p] [-o out] pkg[==ver] [...]
options:
-c | --chainguard use libraries.cgr.dev [default]
-p | --pypi use pypi
-o | --output DIR write output to DIR [default .]
-v | --verbose increase verbosity
EOF
}
VERBOSITY=0
TEMP_D=""
stderr() { echo "$@" 1>&2; }
fail() { local r=$?; [ $r -eq 0 ] && r=1; failrc "$r" "$@"; }
failrc() { local r="$1"; shift; [ $# -eq 0 ] || stderr "$@"; exit "$r"; }
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || stderr "$@"; return 1; }
cleanup() {
[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
debug() {
local level="$1"
shift
[ "${level}" -gt "${VERBOSITY}" ] && return
stderr "${@}"
}
main() {
local short_opts="hcdo:pv"
local long_opts="help,chainguard,deps,pypi,output:,verbose"
local getopt_out=""
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
{ bad_Usage; return; }
local cur="" next="" index="" index_name="" index_print="" output="."
local depsflag="--no-deps"
while [ $# -ne 0 ]; do
cur="$1"
next="$2"
case "$cur" in
-h|--help) Usage ; exit 0;;
-c|--chainguard) index_name="cgr";;
-o|--output) output="$next";;
-d|--deps) depsflag="";;
-p|--pypi) index_name="pypi";;
-v|--verbose) VERBOSITY=$((VERBOSITY+1));;
--) shift; break;;
esac
shift;
done
[ $# -ne 0 ] || { bad_Usage "must provide arguments"; return; }
[ -n "$index_name" ] || index_name=cgr
case "$index_name" in
pypi)
index="https://pypi.org/simple"
index_print="$index"
;;
cgr)
local tok=""
if [ -n "$AUTH_TOK" ]; then
tok="$AUTH_TOK"
else
out=$(chainctl auth token --audience="libraries.cgr.dev") ||
fail "could not get token for libraries.cgr.dev"
tok="$out"
fi
index="https://user:$tok@libraries.cgr.dev/python/simple"
index_print="https://libraries.cgr.dev/python/simple"
;;
esac
if [ $((VERBOSITY)) -gt 0 ]; then
stderr "using index $index_print"
fi
TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
fail "failed to make tempdir"
trap cleanup EXIT
local vdir="$TEMP_D/venv" out="" py3=""
out=$(uv venv "$vdir" 2>&1) || { stderr "$out"; fail "uv venv failed"; }
# shellcheck disable=SC1091
out=$(
export UV_LINK_MODE=copy
. "$vdir/bin/activate" &&
uv pip install pip 2>&1
) || fail "uv pip install pip failed: $out"
py3="$vdir/bin/python3"
out=$(cd "$TEMP_D" &&
"$py3" -m pip download ${depsflag:+"$depsflag"} --only-binary=:all: \
--index-url="$index" "$@" 2>&1) || {
stderr "$out"
fail "[$index_name] failed from $index pip download $*"
}
local wheels=""
wheels=$(cd "${TEMP_D}" && echo *.whl)
[ "$wheels" = "*.whl" ] &&
fail "no wheels were downloaded"
mv "${TEMP_D}"/*.whl "$output" ||
fail "failed mv wheel(s) to $output"
stderr "[$index_name] wrote to $output: $wheels"
return 0
}
main "$@"
Raw
git-spush
#!/bin/sh
# shellcheck disable=SC3043
Usage() {
cat <<EOF
${0##*/} [-f|--force] remote [thing]
This is just a very simplistic wrapper around 'git push'
that attempts to sign things before pushing if not already signed.
Use it by disabling git commit signing by default
git config --local commit.gpgsign false
And then just using this wrapper around 'git push' instead
of 'git push'.
git spush origin
This will sign the top commit of not signed and then push.
EOF
}
fail() { [ $# -eq 0 ] || msg "$@"; exit 1; }
msg() { echo "[$0]" "$@" 1>&2; }
checkrun() {
local rc=""
"$@" && return 0
rc=$?
msg "failed [$rc] running $*"
return $rc
}
spush() {
local remote="" out="" refspec="" branch="" force=""
case " $* " in
*\ --force\ *)
# shellcheck disable=SC2046
set -- $(echo "$@" | sed 's,--force,,')
force="--force";;
esac
[ $# -gt 0 ] || fail "must have a remote"
remote="$1"
shift
if [ $# -eq 0 ]; then
set -- "HEAD"
elif [ $# -gt 1 ]; then
fail "cannot handle multiple refspecs"
fi
refspec="$1"
branch=$(checkrun git branch --show-current) || return
case "$refspec" in
HEAD|"$branch") :;;
*) fail "do not know how to handle push refspec $refspec";;
esac
out=$(checkrun git log --pretty="format:%G?" --max-count=1 "$refspec") ||
return
case "$out" in
N)
msg "$refspec: signing"
checkrun git commit --amend --no-edit --gpg-sign || return
;;
*) msg "$refspec: already signed [$out]";;
esac
git push ${force:+"$force"} "$remote" "$refspec"
}
spush "$@"
Raw
local-with-auth
#!/bin/sh
# shellcheck disable=SC3043
fail() { echo "$@" 1>&2; exit 1; }
token() {
local audience="$1" myuid=""
set -- chainctl auth token --audience="$audience"
[ -z "${SUDO_UID}" ] && { "$@"; return; }
myuid=$(id -u) || { echo "id -u failed. seriously" 1>&2; return 1; }
[ "$myuid" = "0" ] && set -- sudo "--user=#${SUDO_UID}" "$@"
"$@"
}
withenv() {
local env="$1"
shift
if [ -n "$SUDO" ]; then
set -- "$SUDO" "$env" "$@"
[ "$verbose" = "true" ] && echo "$@" | sed -e "s/$tok/<token>/" 1>&2
exec "$@"
fi
if [ "$verbose" = "true" ]; then
echo "$env" "$@" | sed -e "s/$tok/<token>/" 1>&2
fi
# shellcheck disable=SC2163
eval "$env" exec '"$@"'
}
main() {
local SUDO="" audience="${AUDIENCE:-apk.cgr.dev}" verbose="false"
if [ "$1" = "-v" ]; then
verbose=true
shift
fi
tok=$(token "$audience") || fail "failed to get token for AUDIENCE=$audience"
if [ "${1##*/}" = "sudo" ]; then
SUDO="$1"
shift
fi
case "$1" in
curl|*/curl)
p="$1"; shift
set -- "$p" --user "user:$tok" "$@"
[ "$verbose" = "true" ] && echo "$@" | sed -e "s/$tok/<token>/" 1>&2
exec "$@"
;;
apk|*/apk)
withenv HTTP_AUTH="basic:$audience:user:$tok" "$@";;
esac
# unknown. just put AUTH_TOK in environment.
withenv AUTH_TOK="$tok" "$@"
}
main "$@"
Raw
netrc-update
#!/bin/sh
# shellcheck disable=SC3043,SC2162
CR='
'
fail() { stderr; exit 1; }
stderr() { echo "$@" 1>&2; }
update() {
local quiet="false"
[ "$1" = "--quiet" ] && quiet=true && shift
local netrc="${1:-$HOME/.netrc}" cached="$2" bd="" d="" tok=""
for d in "$cached"/*; do
bd=${d##*/}
case "$bd" in
https:*) continue;;
*.cgr.dev) :;;
*) continue;;
esac
[ -f "$d/oidc-token" ] || continue
tok=""
if read tok < "$d/oidc-token" || [ -n "$tok" ]; then
buf="${buf}machine $bd${CR}login token${CR}password $tok${CR}${CR}"
fi
done
[ "$quiet" = "true" ] || echo "update $netrc"
( umask 066 && printf "%s" "$buf" > "$netrc" )
}
watch() {
local netrc="${1:-$HOME/.netrc}" cached="${2:-$HOME/.cache/chainguard}"
command -v inotifywait >/dev/null ||
fail "no inotifywait command"
[ -d "$cached" ] || mkdir -p "$cached" ||
fail "could not create $cached"
echo "updating $netrc from $cached first"
update --quiet "$netrc" "$cached" ||
fail "initial update failed"
echo "watching $cached and updating $netrc"
while :; do
out=$(inotifywait -r -e create,modify,delete "$cached" 2>&1) || {
local rc=$?
printf "%s\n" "$out" 1>&2
fail "fail[$rc] inotifywait -r -e create,modify,delete $cached"
}
"$0" update --quiet "$netrc" "$cached" ||
fail "failed to update $netrc"
echo "$(date)" "updated $netrc"
done
}
Usage_systemd_setup() {
cat <<EOF
Usage: ${0##*/} systemd-setup
setup systemd job to run $0 by writing
and enabling ~/.config/systemd/user/netrc-update.service
EOF
}
systemd_setup() {
if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
exit
fi
[ $# -eq 0 ] || { Usage 1>&2; exit 1; }
local service="netrc-update.service" d="$HOME/.config/systemd/user"
[ -d "$d" ] || mkdir -p "$d" ||
fail "could not create dir $d"
cat > "$d/$service" <<EOF
[Unit]
Description=Watch chainctl cache and update netrc
[Service]
Type=simple
ExecStart=$0 watch
[Install]
WantedBy=default.target
EOF
# shellcheck disable=SC2181
[ $? -eq 0 ] || fail "failed writing $d/$service"
stderr "wrote $d/$service"
systemctl --user enable "$service" ||
fail "failed enabling $service"
systemctl --user daemon-reload ||
fail "failed systemctl daemon-reload"
stderr "systemctl daemon-reload'd"
systemctl --user start "$service" ||
fail "failed starting $service"
stderr "enabled $service"
}
case "$1" in
watch|update) n="$1"; shift; $n "$@"; exit;;
systemd-setup) shift; systemd_setup "$@"; exit;;
esac
update "$@"
Raw
pypi-info
#!/usr/bin/env -S uv run --script
# /// script
# requires-python = ">=3.14"
# dependencies = [
# "packaging",
# ]
# ///
"""Query and download packages from a PEP 503/691 simple index.
By default the client speaks the PEP 691 JSON simple API (sending the
``application/vnd.pypi.simple.v1+json`` Accept header), which works against both
pypi.org and libraries.cgr.dev. The legacy ``/<pkg>/json`` path API is no longer
used (libraries.cgr.dev does not serve it). Pass --simple to use the HTML simple
API instead.
"""
from __future__ import annotations
import argparse
import base64
import html.parser
import json
import netrc as netrc_module
import os
import re
import sys
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime
from pathlib import Path
from typing import Any, Optional
from packaging.utils import InvalidSdistFilename, InvalidWheelFilename, parse_sdist_filename, parse_wheel_filename
from packaging.version import InvalidVersion, Version
DEFAULT_PYPI_URL = os.environ.get("PYPI_URL", "https://pypi.org/simple")
ECO_URL = "https://libraries.cgr.dev/python/simple"
ACCEPT_JSON = "application/vnd.pypi.simple.v1+json"
ACCEPT_HTML = "application/vnd.pypi.simple.v1+html"
def split_url_auth(url: str) -> tuple[str, Optional[tuple[str, str]]]:
"""Split any 'user:password@' credentials out of a URL.
urllib neither uses userinfo in a URL for authentication nor strips it before
connecting, so a 'user:token@host' URL crashes http.client's host parsing.
Pulling the credentials out here lets us support the convenient inline form
while keeping the secret out of the URL we connect to or log.
Returns (clean_url, (username, password)) if userinfo was present, else
(url, None). Username and password are percent-decoded.
"""
parsed = urllib.parse.urlsplit(url)
if parsed.username is None:
return url, None
host = parsed.hostname or ""
if parsed.port is not None:
host = f"{host}:{parsed.port}"
clean_url = parsed._replace(netloc=host).geturl()
username = urllib.parse.unquote(parsed.username)
password = urllib.parse.unquote(parsed.password) if parsed.password else ""
return clean_url, (username, password)
def get_auth(
netrc_file: Optional[str], url: str, inline_auth: Optional[tuple[str, str]] = None
) -> Optional[tuple[str, str]]:
"""Return (username, password) for the given URL, or None.
Priority:
1. Credentials embedded in the URL (user:password@host)
2. --netrc FILE (entry for the URL's host)
3. AUTH_TOKEN env var ('user:pass' or a bare token with user 'token')
4. ~/.netrc (entry for the URL's host)
"""
if inline_auth:
return inline_auth
host = urllib.parse.urlparse(url).hostname or ""
if netrc_file:
nrc = netrc_module.netrc(netrc_file)
auth = nrc.authenticators(host)
if auth:
login, _, password = auth
return (login or "token", password or "")
token = os.environ.get("AUTH_TOKEN")
if token:
if ":" in token:
username, _, password = token.partition(":")
return (username, password)
return ("token", token)
default_netrc = os.path.expanduser("~/.netrc")
if os.path.exists(default_netrc):
nrc = netrc_module.netrc(default_netrc)
auth = nrc.authenticators(host)
if auth:
login, _, password = auth
return (login or "token", password or "")
return None
def _build_request(url: str, auth: Optional[tuple[str, str]], accept: Optional[str] = None) -> urllib.request.Request:
"""Build a urllib Request with optional HTTP Basic auth and Accept header."""
req = urllib.request.Request(url)
if auth:
username, password = auth
credentials = base64.b64encode(f"{username}:{password}".encode()).decode()
req.add_header("Authorization", f"Basic {credentials}")
if accept:
req.add_header("Accept", accept)
return req
class _AuthStrippingRedirectHandler(urllib.request.HTTPRedirectHandler):
"""Drop the Authorization header when a redirect crosses to another host.
A download URL on the index host (e.g. libraries.cgr.dev) commonly 302s to a
pre-signed object-store URL; forwarding our index credentials there both
leaks the secret and is rejected (HTTP 400) by the signed-URL host.
"""
def redirect_request(self, req, fp, code, msg, headers, newurl):
new = super().redirect_request(req, fp, code, msg, headers, newurl)
if new is not None:
old_host = urllib.parse.urlparse(req.full_url).hostname
new_host = urllib.parse.urlparse(newurl).hostname
if old_host != new_host:
for key in list(new.headers):
if key.lower() == "authorization":
del new.headers[key]
return new
_OPENER = urllib.request.build_opener(_AuthStrippingRedirectHandler())
def _urlopen(req: urllib.request.Request):
return _OPENER.open(req)
def fetch_json(url: str, auth: Optional[tuple[str, str]] = None) -> dict[str, Any]:
"""Fetch a PEP 691 JSON document from a simple-index URL."""
req = _build_request(url, auth, ACCEPT_JSON)
try:
with _urlopen(req) as response:
return json.loads(response.read().decode())
except urllib.error.HTTPError as e:
print(f"Error fetching {url}: {e.code}", file=sys.stderr)
sys.exit(1)
except Exception as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
class _LinkParser(html.parser.HTMLParser):
"""Parse anchor links from PEP 503 simple index HTML."""
def __init__(self) -> None:
super().__init__()
self.links: list[tuple[str, str]] = [] # (href, text)
self._current_href: Optional[str] = None
def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None:
if tag == "a":
for name, value in attrs:
if name == "href" and value:
self._current_href = value
break
def handle_data(self, data: str) -> None:
if self._current_href is not None:
self.links.append((self._current_href, data.strip()))
def handle_endtag(self, tag: str) -> None:
if tag == "a":
self._current_href = None
def fetch_html_files(url: str, auth: Optional[tuple[str, str]] = None) -> list[dict[str, Any]]:
"""Fetch an HTML simple-index page and return file dicts (filename, url)."""
req = _build_request(url, auth, ACCEPT_HTML)
try:
with _urlopen(req) as response:
content = response.read().decode("utf-8", errors="replace")
except urllib.error.HTTPError as e:
print(f"Error fetching {url}: {e.code}", file=sys.stderr)
sys.exit(1)
except Exception as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
parser = _LinkParser()
parser.feed(content)
files = []
for href, text in parser.links:
if not text:
continue
absolute_url = urllib.parse.urljoin(url, href.split("#")[0])
files.append({"filename": text, "url": absolute_url})
return files
# Fallback for artifacts packaging can't parse (eggs, exes), e.g.
# Pillow-6.1.0-py2.7-win32.egg, package-1.0.exe
_artifact_regex = re.compile(r"(?P<pkg>.*)-(?P<ver>[0-9][0-9.]*)(?:[-.][-\w.]*)?\.(?P<ext>[a-zA-Z]+)")
def parse_version(filename: str) -> Optional[Version]:
"""Parse a Version out of an artifact filename, or None if unrecognized."""
try:
_, ver, _, _ = parse_wheel_filename(filename)
return ver
except InvalidWheelFilename:
pass
except InvalidVersion:
return None
try:
_, ver = parse_sdist_filename(filename)
return ver
except (InvalidSdistFilename, InvalidVersion):
pass
m = _artifact_regex.search(filename)
if m:
try:
return Version(m.group("ver"))
except InvalidVersion:
return None
return None
def parse_filename(filename: str) -> tuple[str, Version]:
try:
pkg, ver, _, _ = parse_wheel_filename(filename)
return pkg, ver
except InvalidWheelFilename:
return parse_sdist_filename(filename)
def fetch_files(args: argparse.Namespace, pkg: str, auth: Optional[tuple[str, str]]) -> list[dict[str, Any]]:
"""Return the list of file dicts for a package in the configured mode.
JSON mode dicts carry filename/url/size/upload-time; HTML mode only
filename/url.
"""
pkg_url = f"{args.url.rstrip('/')}/{pkg}/"
if args.simple:
return fetch_html_files(pkg_url, auth)
data = fetch_json(pkg_url, auth)
return data.get("files", [])
def files_to_rows(files: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Normalize PEP 691 file dicts into display rows, parsing version from name."""
rows = []
for f in files:
filename = f["filename"]
ver = parse_version(filename)
if ver is None:
print(f"Skipping unrecognized artifact {filename}", file=sys.stderr)
continue
rows.append(
{
"version": str(ver),
"_version": ver,
"upload_time": f.get("upload-time"),
"filename": filename,
"size": f.get("size"),
"url": f.get("url", ""),
}
)
return rows
def cmd_ls(args: argparse.Namespace) -> None:
"""List releases and files for a package (optionally filtered by version)."""
auth = get_auth(args.netrc, args.url, args.inline_auth)
rows = files_to_rows(fetch_files(args, args.package, auth))
if args.version:
try:
want = Version(args.version)
except InvalidVersion:
print(f"Error: invalid version {args.version!r}", file=sys.stderr)
sys.exit(1)
rows = [r for r in rows if r["_version"] == want]
if args.long:
compute_deltas(rows)
rows.sort(key=lambda r: (r["_version"], r["delta"], r["filename"]))
print_rows(rows)
else:
rows.sort(key=lambda r: (r["_version"], r["filename"]))
for row in rows:
print(row["filename"])
def compute_deltas(rows: list[dict[str, Any]]) -> None:
"""Add a delta field (seconds from the oldest upload within each version)."""
if not rows:
return
versions: dict[str, list[dict[str, Any]]] = {}
for row in rows:
versions.setdefault(row["version"], []).append(row)
for version_rows in versions.values():
timestamps = []
for row in version_rows:
iso = row["upload_time"]
if iso:
dt = datetime.fromisoformat(iso.replace("Z", "+00:00"))
timestamps.append(int(dt.timestamp()))
else:
timestamps.append(None)
known = [t for t in timestamps if t is not None]
oldest = min(known) if known else 0
for row, ts in zip(version_rows, timestamps):
row["delta"] = (ts - oldest) if ts is not None else 0
iso = row["upload_time"]
row["upload_time"] = iso.split(".")[0] if iso else "-"
def print_rows(rows: list[dict[str, Any]]) -> None:
"""Print rows in the format: version upload_time delta filename size"""
for row in rows:
size = row["size"] if row["size"] is not None else "-"
print(f"{row['version']} {row['upload_time']} {row['delta']} {row['filename']} {size}")
def cmd_json(args: argparse.Namespace) -> None:
"""Print the raw PEP 691 JSON for a package (optionally filtered by version)."""
auth = get_auth(args.netrc, args.url, args.inline_auth)
url = f"{args.url.rstrip('/')}/{args.package}/"
data = fetch_json(url, auth)
if args.version:
try:
want = Version(args.version)
except InvalidVersion:
print(f"Error: invalid version {args.version!r}", file=sys.stderr)
sys.exit(1)
data["files"] = [
f for f in data.get("files", []) if parse_version(f["filename"]) == want
]
json.dump(data, sys.stdout, indent=2)
sys.stdout.write("\n")
def cmd_get(args: argparse.Namespace) -> None:
"""Download a file by name."""
filename = args.filename
output = args.output or filename
try:
pkg, _ = parse_filename(filename)
except (InvalidWheelFilename, InvalidSdistFilename):
print(f"Error: Could not parse package/version from {filename}", file=sys.stderr)
sys.exit(1)
auth = get_auth(args.netrc, args.url, args.inline_auth)
files = fetch_files(args, pkg, auth)
file_url = None
for f in files:
if f["filename"] == filename:
file_url = f["url"]
break
if not file_url:
print(f"Error: File {filename} not found in {pkg} index", file=sys.stderr)
sys.exit(1)
if Path(output).is_dir():
output = Path(output) / filename
# Only send the Basic auth header to the download URL if it shares the index
# host; never leak index credentials to a third-party file host.
file_host = urllib.parse.urlparse(file_url).hostname or ""
index_host = urllib.parse.urlparse(args.url).hostname or ""
file_auth = auth if file_host == index_host else None
req = _build_request(file_url, file_auth)
try:
with _urlopen(req) as response, open(output, "wb") as fh:
while True:
chunk = response.read(65536)
if not chunk:
break
fh.write(chunk)
print(f"wrote {output}", file=sys.stderr)
except Exception as e:
print(f"Error downloading {file_url} to {output}: {e}", file=sys.stderr)
sys.exit(1)
def main() -> None:
parser = argparse.ArgumentParser(
prog="pypi-info",
description="Query and download packages from a PEP 503/691 simple index",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=f"""
Default URL: {DEFAULT_PYPI_URL} (override with PYPI_URL env, --url, or --eco)
By default the PEP 691 JSON simple API is used (Accept:
{ACCEPT_JSON}). Pass --simple to use the HTML simple API instead.
Authentication (HTTP Basic; checked in order):
1. Credentials embedded in the URL (https://user:password@host/...)
2. Netrc file specified via --netrc
3. AUTH_TOKEN environment variable ('user:pass' or bare token with user 'token')
4. ~/.netrc (entry for the URL's host)
""",
)
url_group = parser.add_mutually_exclusive_group()
url_group.add_argument(
"-u", "--url",
default=DEFAULT_PYPI_URL,
metavar="URL",
help=f"Base simple-index URL (default: {DEFAULT_PYPI_URL})",
)
url_group.add_argument(
"-E", "--eco",
dest="url",
action="store_const",
const=ECO_URL,
help=f"shortcut for --url {ECO_URL}",
)
parser.add_argument(
"-n", "--netrc",
metavar="FILE",
help="Netrc file for authentication (default: ~/.netrc)",
)
parser.add_argument(
"--simple",
action="store_true",
help="Use the HTML simple API instead of PEP 691 JSON (incompatible with -l)",
)
subparsers = parser.add_subparsers(dest="command", required=True)
# ls subcommand
ls_parser = subparsers.add_parser("ls", help="List releases and files")
ls_parser.add_argument("package", help="Package name")
ls_parser.add_argument("version", nargs="?", help="Specific version (optional)")
ls_parser.add_argument("-l", "--long", action="store_true", help="Long format output")
# json subcommand
json_parser = subparsers.add_parser("json", help="Print raw PEP 691 JSON")
json_parser.add_argument("package", help="Package name")
json_parser.add_argument("version", nargs="?", help="Specific version (optional)")
# get subcommand
get_parser = subparsers.add_parser("get", help="Download a file by name")
get_parser.add_argument("filename", help="Filename to download")
get_parser.add_argument("output", nargs="?", help="Output path (optional)")
args = parser.parse_args()
# Pull any 'user:password@' credentials out of the URL so the secret never ends
# up in the URLs we connect to, join against, or log; route them to get_auth.
args.url, args.inline_auth = split_url_auth(args.url)
if args.simple and getattr(args, "long", False):
parser.error("--long is incompatible with --simple (HTML lacks size/upload-time)")
if args.simple and args.command == "json":
parser.error("the 'json' command requires the JSON API; do not use --simple")
if args.command == "ls":
cmd_ls(args)
elif args.command == "json":
cmd_json(args)
elif args.command == "get":
cmd_get(args)
if __name__ == "__main__":
main()
Raw
test-installable
#!/bin/sh
# shellcheck disable=SC2015,SC2039,SC2166,SC3043
VERBOSITY=0
TEMP_D=""
stderr() { echo "$@" 1>&2; }
fail() { local r=$?; [ $r -eq 0 ] && r=1; failrc "$r" "$@"; }
failrc() { local r="$1"; shift; [ $# -eq 0 ] || stderr "$@"; exit "$r"; }
Usage() {
cat <<EOF
Usage: ${0##*/} [ options ] packages
Taken partially from wolfi-dev/os/.github/workflows/ci-build.yaml
"Check that packages can be installed with apk add"
options:
-h | --help help
-v | --verbose increase verbosity
Example:
${0##*/} --repo=packages packages/*.apk
EOF
}
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || stderr "$@"; return 1; }
cleanup() {
[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
debug() {
local level="$1"
shift
[ "${level}" -gt "${VERBOSITY}" ] && return
stderr "${@}"
}
inside_Usage() {
cat <<EOF
Usage: ${0##*/} inside repo [ repo ... ] -- packages
EOF
}
vr() {
"$@" && return 0;
fail "FAILED[$?]: $*";
}
runv() {
local v="$1"
shift
debug "$v" "execute:" "$@"
"$@"
}
inside() {
local short_opts="hr:v"
local long_opts="help,repo:,verbose"
local getopt_out=""
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
{ bad_Usage; return; }
local cur="" next=""
while [ $# -ne 0 ]; do
# shellcheck disable=SC2034
{ cur="$1"; next="$2"; }
case "$cur" in
-h|--help) Usage ; exit 0;;
-v|--verbose) VERBOSITY=$((VERBOSITY+1));;
--) shift; break;;
esac
shift;
done
for r in /repos/repo*; do
[ -d "$r" ] || continue
rflags="$rflags --repository=$r"
done
rflags=${rflags# }
ls /repos/
if [ $# -eq 0 ]; then
[ -d /repos/repo0 ] ||
fail "sorry, no /repos/repo0 and no packages"
set -- $(find /repos/repo0 -name "*.apk" -type f)
fi
eroot="/tmp/emptyroot"
vr mkdir -p "$eroot/etc/apk"
vr cp -r "/etc/apk"/* "$eroot/etc/apk"
: > "$eroot/etc/apk/world"
vr mkdir -p "$eroot/lib/apk/db"
vr touch "$eroot/lib/apk/db"/{installed,lock,scripts.tar,triggers}
vr mkdir -p "$eroot" "$eroot/var/cache/apk"
vr apk update "--root=$eroot"
fails=""
n=$#
for f in "$@"; do
echo "== $f =="
vr tar -Oxf "$f" .PKGINFO
apk add --allow-untrusted --simulate \
--root="$eroot" ${rflags} \
"$f"
[ $? -eq 0 ] || fails="$fails $f"
echo
done
echo "tested $# apk installs"
[ -z "$fails" ] && exit 0
set +f
set -- $fails
echo "$# failed install: $fails"
exit 1
}
fullpath() {
local p="$1"
( cd "$p" && pwd )
}
main() {
local short_opts="hr:v"
local long_opts="help,repo:,verbose"
local getopt_out=""
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
{ bad_Usage; return; }
local cur="" next="" repos="" r=""
while [ $# -ne 0 ]; do
# shellcheck disable=SC2034
{ cur="$1"; next="$2"; }
case "$cur" in
-h|--help) Usage ; exit 0;;
-r|--repo)
r=$(fullpath "$next") ||
fail "failed to get full path to $next"
repos="$repos $r"; shift;;
-v|--verbose) VERBOSITY=$((VERBOSITY+1));;
--) shift; break;;
esac
shift;
done
repos=${repos# }
if [ -z "$repos" ]; then
[ -d ./packages ] ||
fail "no --repo given and default (./packages) is not a dir"
repos="$PWD/packages"
fi
debug 1 "repos: $repos"
set +f
local volumes="" r="" n=0
volumes=""
for r in ${repos}; do
volumes="$volumes --volume=$r:/repos/repo$n"
n=$((n+1))
done
volumes=${volumes# }
runv 1 docker-run-action \
$volumes \
--image=cgr.dev/chainguard/wolfi-base:latest \
--entrypoint=/bin/sh -- -s "inside" "$@" <"$0"
}
if [ "$1" = "inside" ]; then
shift
inside "$@"
fi
main "$@"
# vi: ts=4 expandtab
Raw
usrmerge-data
#!/bin/bash
#
# This was built to grab a list of packages that needed to be updated
# for usrmerge. It outputs 3 different files, each with
# a list of repo, pkgname, origin, version, B, S, US, LIB
# where B=number of files in bin/
# S=number of files in sbin/
# US=number of files in usr/sbin/
# LIB=number of files in lib/
#
# As of 2025-04-01 there are no files in wolfi, enterprise or extra
# in bin/, sbin/, or usr/sbin/.
set -o pipefail
repo_info() {
local name="$1" infod="$2" gitd="$3"
shift 3
(
cd "$infod" || exit 1
if [ $# -eq 0 ]; then
set -- *.info
fi
for info in "$@"; do
flist=${info%.info}.flist
out=$(awk '
$1 == "origin" { o=$3; };
$1 == "pkgname" { p=$3; };
$1 == "pkgver" { v=$3 };
END { printf("%s\t%s\t%s\n", o, v, p); }' "$info") || { echo "failed"; exit 1; }
set -- $out
origin=$1
pkg=$2
ver=$3
if [ ! -f "$gitd/$origin.yaml" ]; then
echo "[$name] skipping $origin/$pkg [obsolete/not in git]" 1>&2
continue
fi
out=$(awk '
$6 ~ mus { us=us+1; }
$6 ~ mb { b=b+1; }
$6 ~ ms { s=s+1; }
$6 ~ mlib { lib=lib+1; }
END { printf("%d %d %d %d\n", b, s, us, lib); }' \
mus="^usr/sbin/" mb="^bin/" ms="^sbin/" mlib="^lib/" \
"$flist") || exit 1
set -- $out
b=$1
s=$2
us=$3
lib=$4
[ "$b" = "0" ] && [ "$s" = "0" ] && [ "$us" = "0" ] && [ "$lib" = "0" ] && continue
printf "%s\t%s\t%s\t%s\t%d\t%d\t%d\t%d\n" \
"$name" "$origin" "$ver" "$pkg" "$b" "$s" "$us" "$lib"
done
)
}
outd=${1:-/tmp}
mkdir -p "$outd" || exit 1
repos="extras enterprise wolfi"
for repo in $repos; do
case $repo in
enterprise) upstream="git@github.com:chainguard-dev/enterprise-packages";;
extras) upstream="git@github.com:chainguard-dev/extra-packages";;
wolfi) upstream="git@github.com:wolfi-dev/os";;
esac
if [ ! -d "$repo.git" ]; then
git clone -o upstream "$upstream" "$repo.git" || { rm -Rf $repo.git; exit 1; }
else
( cd "$repo.git" && git fetch upstream && git checkout upstream/main ) ||
{ echo "failed update $repo.git"; exit 1; }
fi
done
for repo in $repos; do
get-archive-info "--archive=$repo" "$PWD/$repo" ||
{ echo "failed get-archive-info --archive=$repo $PWD/$repo"; exit 1; }
repo_info "$repo" "$PWD/$repo" "$PWD/$repo.git" > "$outd/$repo-usrmerge.txt" ||
exit 1
done
sort -k2 $outd/*-usrmerge.txt > $outd/all.txt
Raw
wolfi-base
#!/bin/bash
# shellcheck disable=SC2015,SC2039,SC2166,SC3043
VERBOSITY=0
TEMP_D=""
IMAGE=${IMAGE:-"cgr.dev/chainguard-private/chainguard-base:latest"}
WOLFI_IMAGE="cgr.dev/chainguard/wolfi-base:latest"
stderr() { echo "$@" 1>&2; }
fail() { local r=$?; [ $r -eq 0 ] && r=1; failrc "$r" "$@"; }
failrc() { local r="$1"; shift; [ $# -eq 0 ] || stderr "$@"; exit "$r"; }
cleanup() {
[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
Usage() {
cat <<EOF
Usage: ${0##*/} [ options ] <<ARGUMENTS>>
run wolfi-base in docker
options:
-A | --arch A create wolfi-base for arch A (amd64/arm64)
-e | --enterprise add repo for enterprise
-x | --extras add extras repo
-W | --wolfi use wolfi image ${WOLFI_IMAGE}
default: ${IMAGE}
image can also be set with env IMAGE
-s | --stereo add stereo (env STEREO_D or ~/src/stereo)
-r | --repo-dir D add repo dir D. can be either
1. dir with a packages/<arch>/APKINDEX>.tar.gz
and one *.rsa.pub
2. a path to a packages/ dir whose parent has
one *.rsa.pub
-h | --help Usage
--mount M add --mount=M
-v | --volume V add --volume=V
--entrypoint N add --entrypoint=N
-w | --workdir W add --workdir=W
-V | --verbose increase verbosity
--user U add --user=U
--batch non-interactive (do not use -it)
--keep keep (do not use --rm)
-P --priv run privliged container
EOF
}
shellquote(){
python3 -c 'import shlex, sys; print("$ " + " ".join([shlex.quote(a) for a in sys.argv[1:]]))' "$@"
}
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || stderr "$@"; return 1; }
debug() {
local level="$1"
shift
[ "${level}" -gt "${VERBOSITY}" ] && return
stderr "${@}"
}
repos() {
local enterprise="$1" extras="$2"
echo "https://apk.cgr.dev/chainguard"
[ "$enterprise" = "true" ] &&
echo "https://apk.cgr.dev/chainguard-private"
[ "$extras" = "true" ] &&
echo "https://packages.cgr.dev/extras"
return 0
}
keys() {
# $ docker run --entrypoint=/bin/sh --rm -it
# cgr.dev/chainguard-private/chainguard-base:latest
# -c 'cd /etc/apk/keys && for k in chainguard-*.pub; do echo == $k ==; cat $k; done'
local dir="$1" enterprise="$2" extras="$3"
if [ "$enterprise" = "true" ]; then
cat > "$dir/chainguard-ed520f52bd1d4cab06899f4b3cee067d1d9dda408f181e818227372a4757b0b8.rsa.pub" <<EOF
-----BEGIN PUBLIC KEY-----
MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAphF6T7nrZv3CpCP/0QtZ
KhjnVCsGv3UvLpZakfApuTmyhEAXbACiBRN/Raq7/5Mb7WaDZSUbm0kt/NsED1MU
9owXaa7l257kX6Em50pm3d7bvlji7lXJACCMObSViW8w5+tBdXeROR6F2MsIFF9R
hEVoY11gJw7+m+z4sjZTC2E5jX0QwEokwrYrcKRMVEtyan+aOnHL/2khSpEJrjtL
0I/zXUK1Fghj3htWdQAKFK4HVGTcSEGjcgh+2MJzEGaLPQOpw0o6x5AQtuvrqGVE
ySAKqXlfpDIOcoi5gXGvOzn9p7OhSf5pLGr4J6QcghUuLlBJJgfMqO8mwv+eW6uq
Ss93UIWycdf6IzbN6yCBXXFhiSXQ+0T4u7y/o58GW28jQ5CwsUXACuX9KdJTMWSa
uyk7pEALLTlR6nrWULvcRbLjvO+P6xSEywxC2KsHMYnpTp4rP25Mm2/lfI/qV5Xj
B93hlEdHXiNsSUjYlSRVJbgGWKq2pAPqaCIPeFFFjmOrj1t1e1GJt4cuPaZZmTrX
REJW8++bVHidfg/EXJnxHiAlWmu1Nruv3xaI4F0U9hTh6ber6wl6erl+bMlBlTPo
f7UHP0Zc5XkR1oPkhxxfKnNleqmKg4RmoyBWrZeFkYdthRSbzH0yl5pdyL8q2ujt
mtYzK5KoeklsR7MjC+2DZ+UCAwEAAQ==
-----END PUBLIC KEY-----
EOF
cat > "$dir/chainguard-f7fe2e42ceb7963e8dc4117e6bba19516ef3c4c7d86cc684404461d942342e93.rsa.pub" <<EOF
-----BEGIN PUBLIC KEY-----
MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEApPUZzx7q16nGmlJUdFBs
W36H9S4HtBzA83FSGBiltzl3W9uesH0elFLjJIhNL8nYY7y+vQgIk3e506ISxlMk
ZUWEx7/CCS4r1p81yaCRywFuaMRfBcDJI4h/y5scjyHUb9qmI/EtHCv1KrGnTppe
mHkUeNCvLkSZYjo4/QoLSw0PLtW06Xj36pMHugDh83q+NUKXgnD83wRmpUaAKH8i
Btmk63oqQHvFirxdAj8JMdClHHHDQRv059k7lnjAne5RPHZY6vV89CIBil0GcII+
Yj6sOjwMPNDv8ILItpvT+ntuQU6roXeu6yiIXRSYy1rTzp1RQI4utWlQoDcTEcCs
nV492t4J+YE6UDGe+QXLtViMvUZjXH2DJbMQvfNemNR/F1ZZjzzFZeBCDyi+hRNy
Wdt1PrJVnFAQzC6z1hMAUrgyz5XS0XIkvQ6/ziuwGXp7Arart0LOT1WRFS0dEFlx
ySk3qyQ1yOLdshbwvZ5OO5IxRrOZPCyWaB5vXshd8QMT4/bNwqt8n27q9CNUfebH
VhTiJo6enTGrEON0OF1u+6aUn/s+/zDQrYMOz4AVa+t4lLykgLnW7ZX3aw3KXTva
/4O4rUx8dtSdhGULXVxknSn1AVHmtY2s0927m9CN3tipCK1Sl+iJLeDEDH+kQHe8
yPqNtAI+bL6cwg/E2yL5KwMCAwEAAQ==
-----END PUBLIC KEY-----
EOF
fi
if [ "$extras" = "true" ]; then
cat > "$dir/chainguard-extras.rsa.pub" <<EOF
-----BEGIN PUBLIC KEY-----
MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxQe5KE+z6aPUl/FUzewy
MogEwmlWYyyUdkd7IIp3EFn3ml8cuC1YgMFhCJH/jjIzp0GShTNWbs9ZHrWUf66Y
fKiOfz970UgjocSx2KOiL1C3C9HlnqRPoUWgvhs7RpYRr4CPJJBtl5zb16BSoA0L
XIxNn/saFzb+DDaTTSg2HuWab2ZJlwI9Hoi6ViEeCLmTcBH+OOm4PfPcHVlBDIXs
Hs3Iwaypbf9txCtH0yynYUKz+IqKT5iSWe2rBEHIG7KE8uf0eNgITFRnqnQ751kI
qBkbyOMcGmAvdvrszY8lEmi8i+NMss0R7KOGLlOUV/U9eBAWq1gykm4ouCSQcoFm
TUJFAhPxZdLWyX2terIegJIpBNj+G7Q6bt9VD7PW+UfoyZbN4pgRk3+5v2QFJEAy
lIrebpc1Ca4xcWAM8GhD9XvHBWVLH9Q3GmQ8CtrVjHFeMrUVAMPmsTELZ0PRqost
MNBBUkyfk46iVi2HoC/sc8thSshDUmgbJs37lEo1/wB+l9l/RDxIn44SCvMWR6GV
jfDXdS+f4xR3D12hFbtInDWgQSek8tlHFCpQDpmtcQ6J38EkN4ZYUvLIoW4u74RG
SHH6dYe1sbDVOd0oxNPttXddRse/XOkHq/4G80AMRU0sytV0e7NauHIyNB4/DI+2
XwqkGfBhFCU7oKtwuixmBH0CAwEAAQ==
-----END PUBLIC KEY-----
EOF
fi
}
find_pubkey() {
local d="$1" ff=""
if ff=$(one_file "$d"/local-*.rsa.pub); then
echo "$ff"
elif ff=$(one_file "$d"/*.rsa.pub); then
echo "$ff"
fi
[ -n "$ff" ]
}
one_file() {
[ $# -eq 1 ] || return 1
[ -f "$1" ]
( cd "$(dirname "$1")" && echo "$PWD/${1##*/}")
}
main() {
local short_opts="A:ehI:Pr:sv:Vw:xW"
local long_opts="arch:,batch,help,enterprise:,entrypoint:,extras:,image:,keep,mount:,priv,user:,stereo,repo-dir:,verbose,volume:,workdir:,wolfi"
local getopt_out=""
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
{ bad_Usage "bad usage"; return; }
local cur="" next=""
local enterprise=false extras=false vols="" mounts="" wolfi=false priv=false uarch=""
local keep=false batch=false stereo=false repods="" entrypoint=""
repods=()
vols=()
mounts=()
while [ $# -ne 0 ]; do
# shellcheck disable=SC2034
{ cur="$1"; next="$2"; }
case "$cur" in
-h|--help) Usage ; exit 0;;
-A|--arch) uarch=$next; shift;;
-e|--enterprise) enterprise=true;;
-x|--extras) extras=true;;
-s|--stereo) stereo=true;;
-r|--repo-dir) repods+=( "$next" );;
--mount) mounts+=( "--mount=$next" );;
-v|--volume) vols+=( "--volume=$next" );;
--entrypoint) entrypoint=$next; shift;;
-w|--workdir) workdir="$next"; shift;;
-V|--verbose) VERBOSITY=$((VERBOSITY+1));;
-I|--image) IMAGE=$next; shift;;
-W|--wolfi) IMAGE=${WOLFI_IMAGE}; wolfi=true;;
--user) user=$next; shift;;
--keep) keep=true;;
--batch) batch=true;;
-P|--priv) priv=true;;
--) shift; break;;
esac
shift;
done
# program starts here
TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
fail "failed to make tempdir"
trap cleanup EXIT
local unamem="" os=linux narch="" arch=""
unamem=$(uname -m) || fail "what? no uname -m"
case "$unamem" in
# native arch
amd64|x86|x86_64) narch="amd64";;
aarch64|arm64) narch="arm64";;
*) fail "unknown uname -m '$unamem'"
esac
arch="$narch"
if [ -n "$uarch" ]; then
case "$uarch" in
amd64|x86|x86_64) arch="amd64";;
aarch64|arm64) arch="arm64";;
*) fail "dont know what arch=$uarch is";;
esac
fi
local hackinteractive="false"
[ "$narch" = "$arch" ] || hackinteractive=true
local mymounts="" tok="" envarg="" reposd="/work/repos"
mymounts=()
audiences="libraries.cgr.dev"
if [ "$enterprise" = "true" ]; then
audiences="$audiences apk.cgr.dev"
fi
(
umask 066
for aud in $audiences ; do
tok=$(chainctl auth token --audience="$aud") ||
{ stderr "failed getting token for $aud"; exit 1; }
printf "%s\n%s\n%s\n\n" "machine $aud" "login token" "password $tok"
done >> "${TEMP_D}/netrc"
) || fail "failed creating netrc"
mymounts+=("--mount=type=bind,source=${TEMP_D}/netrc,destination=/root/.netrc")
repos "$enterprise" "$extras" > "${TEMP_D}/repositories" ||
fail "failed to create repositories"
if [ "$stereo" = "true" ]; then
local n="" found=false
if [ -z "$STEREO_D" ]; then
# shellcheck disable=SC2116
STEREO_D=$(echo ~/src/stereo)
fi
for n in enterprise-packages extra-packages os ; do
if find_pubkey "$STEREO_D/$n" >/dev/null ; then
found=true
repods+=( "stereo/$n:${STEREO_D}/$n" )
fi
done
if [ "$found" != "true" ]; then
fail "did not find any stereo package dirs/keys in STEREO_D - $STEREO_D"
fi
fi
if [ "${#repods[@]}" != 0 ]; then
mkdir -p "${TEMP_D}/keys" ||
fail "failed creating keys in tempdir"
local rd="" pubkey="" pkgd="" name=""
for rd in "${repods[@]}"; do
case "$rd" in
# support name:path
*:*) name=${rd%%:*}; rd=${rd#*:};;
*) name=$(cd "$rd" && d=$PWD && echo "${d##*/}");;
esac
if pubkey=$(find_pubkey "$rd"); then
pkgd=${pubkey%/*}/packages
elif pubkey=$(find_pubkey "$rd/.."); then
pkgd=${pubkey%/*}/packages
else
fail "$rd - did not find a pubkey"
fi
cp "$pubkey" "${TEMP_D}/keys/" ||
fail "failed cp $pubkey -> tempdir/keys"
mymounts+=( "--mount=type=bind,source=$pkgd,destination=$reposd/$name" )
echo "$reposd/$name" >> "${TEMP_D}/repositories"
done
fi
if [ "$wolfi" = "true" ] &&
[ "$enterprise" = "true" -o "$extras" = "true" ]; then
mkdir -p "${TEMP_D}/keys" &&
keys "${TEMP_D}/keys" "$enterprise" "$extras" ||
fail "failed writing keys for wolfi"
fi
local k="" b="" r=""
for k in "${TEMP_D}/keys/"*; do
[ "$k" = "${TEMP_D}/keys/*" ] && continue # no keys
b=${k##*/}
mymounts+=( "--mount=type=bind,source=$k,destination=/etc/apk/keys/$b" )
done
if [ -f "${TEMP_D}/repositories" ]; then
mymounts+=( "--mount=type=bind,source=${TEMP_D}/repositories,destination=/etc/apk/repositories" )
fi
local privops=""
privops=(
--privileged
--security-opt="seccomp=unconfined"
--security-opt="apparmor:unconfined"
)
[ "$priv" = "true" ] || privops=( )
local name="" cmd=() flags=()
[ "$keep" = "false" ] && flags+=("--rm")
[ "$batch" = "true" ] || flags+=("-it")
name=$(petname) || fail "failed to get a name"
stderr "$name eprise=$enterprise extras=$extras priv=$priv arch=$arch $IMAGE"
if [ "$hackinteractive" = "true" ] && [ $# -eq 0 ] && [ -z "$entrypoint" ] &&
[ "$batch" = "false" ]; then
entrypoint=/bin/sh
set -- -il
fi
cmd=(
docker run --rm -it --name="$name"
--platform="$os/$arch"
"${privops[@]}"
"${mymounts[@]}" "${mounts[@]}" "${vols[@]}"
${envarg:+"${envarg}"}
${workdir:+"--workdir=$workdir"}
${entrypoint:+"--entrypoint=${entrypoint}"}
${user:+"--user=$user"}
"${IMAGE}" "$@"
)
[ $VERBOSITY -gt 0 ] && shellquote "${cmd[@]}" 1>&2
"${cmd[@]}"
}
main "$@"
# vi: ts=4 expandtab
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment