Last active
December 5, 2024 00:07
-
-
Save tianon/35528d9851ebf0b0767c04cec5e01bcf to your computer and use it in GitHub Desktop.
A very, *very* basic PoC of using `jq` to manage and manipulate OCI layouts.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
set -Eeuo pipefail | |
oci="$PWD" # TODO parameter/flag (and pass it around to other oci tools correctly!) | |
# ~validate $oci is an OCI layout | |
[ -s "$oci/oci-layout" ] | |
jq -se 'length == 1 and .[0].imageLayoutVersion == "1.0.0"' "$oci/oci-layout" > /dev/null | |
# TODO more/better validation? definitely better error handling | |
IFS=$'\n' # TODO more defensive shell escaping handling | |
descs="$(jq -c '.manifests[]' "$oci/index.json")" | |
set -- $descs | |
referenced= | |
while [ "$#" -gt 0 ]; do | |
desc="$1"; shift | |
file="$(jq <<<"$desc" -r '"blobs/" + (.digest | sub(":"; "/"))')" | |
referenced+="${referenced:+$'\n'}$file" | |
mediaType="$(jq <<<"$desc" -r '.mediaType // ""')" | |
case "$mediaType" in | |
application/vnd.oci.image.index.v1+json | application/vnd.docker.distribution.manifest.list.v2+json) | |
descs="$(oci-get "$desc" | jq -c '.manifests[]')" | |
set -- "$@" $descs | |
continue | |
;; | |
application/vnd.oci.image.manifest.v1+json | application/vnd.docker.distribution.manifest.v2+json) | |
descs="$(oci-get "$desc" | jq -c '.config, .layers[]')" | |
set -- "$@" $descs | |
continue | |
;; | |
# https://github.com/tianon/oci-schema1 | |
application/vnd.docker.distribution.manifest.v1+json | \ | |
application/vnd.docker.distribution.manifest.v1+prettyjws) | |
descs="$(oci-get "$desc" | jq -c '{ mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", digest: .fsLayers[].blobSum }')" | |
set -- "$@" $descs | |
continue | |
;; | |
application/vnd.oci.image.config.v1+json | \ | |
application/vnd.oci.image.layer.v1.tar+gzip | \ | |
application/vnd.docker.container.image.v1+json | \ | |
application/vnd.docker.image.rootfs.diff.tar.gzip | \ | |
application/vnd.docker.image.rootfs.foreign.diff.tar.gzip | \ | |
application/vnd.in-toto+json) | |
# known blob type, no children (TODO *technically* this should happen in the above blocks since layers and config can't have children period, and "artifacts" will make this funky) | |
;; | |
*) | |
echo >&2 "warning: unknown children: $desc" | |
continue | |
;; | |
esac | |
done | |
everything="$(cd "$oci" && find blobs -type f)" | |
comm -13 <(sort -u <<<"$referenced") <(sort -u <<<"$everything") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
set -Eeuo pipefail | |
oci="$PWD" # TODO parameter/flag | |
case "$#" in | |
0) ;; # descriptor on stdin | |
1) exec <<< "$1"; shift ;; # descriptor (now) on stdin | |
*) echo >&2 "error: expected 0 or 1 arguments, got $#" exit 1 ;; | |
esac | |
# ~validate $oci is an OCI layout | |
[ -s "$oci/oci-layout" ] | |
jq -se 'length == 1 and .[0].imageLayoutVersion == "1.0.0"' "$oci/oci-layout" > /dev/null | |
# TODO more/better validation? definitely better error handling | |
# TODO stream of inputs somehow? multi-input? | |
digest="$(jq -sR ' | |
gsub("^[[:space:]]+|[[:space:]]+$"; "") | |
| if startswith("{") and endswith("}") then | |
fromjson | |
elif contains(":") then | |
{ digest: . } | |
else | |
error("input does not look like a descriptor (or digest): " + .) | |
end | |
| .digest | |
' -r)" | |
# TODO also validate checksum/size from provided descriptor/digest? | |
cat "$oci/blobs/${digest/://}" | |
# TODO some way to get the filename directly instead of the contents |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
set -Eeuo pipefail | |
oci="$PWD" # TODO parameter/flag | |
# ~validate $oci is not yet an OCI layout | |
[ ! -e "$oci/oci-layout" ] | |
[ ! -e "$oci/index.json" ] | |
[ ! -e "$oci/blobs" ] | |
# TODO more/better validation? definitely better error handling | |
# https://github.com/opencontainers/image-spec/blob/v1.1.0/image-layout.md | |
jq -nc '{ imageLayoutVersion: "1.0.0" }' > "$oci/oci-layout" | |
jq -n ' | |
{ | |
schemaVersion: 2, | |
mediaType: "application/vnd.oci.image.index.v1+json", | |
manifests: [], | |
} | |
' > "$oci/index.json" | |
mkdir -p "$oci/blobs" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
set -Eeuo pipefail | |
oci="$PWD" # TODO parameter/flag | |
case "$#" in | |
0) ;; # blob data on stdin | |
1) exec < "$1"; shift ;; # blob data (now) on stdin | |
*) echo >&2 "error: expected 0 or 1 arguments, got $#" exit 1 ;; | |
esac | |
# ~validate $oci is an OCI layout | |
[ -s "$oci/oci-layout" ] | |
jq -se 'length == 1 and .[0].imageLayoutVersion == "1.0.0"' "$oci/oci-layout" > /dev/null | |
# TODO more/better validation? definitely better error handling | |
tmp="$(mktemp --directory --tmpdir oci-put.XXXXXXXXXX)" | |
trap "$(printf 'rm -rf %q' "$tmp")" EXIT | |
algo='sha256' | |
sum="$(tee "$tmp/blob-data" | "${algo}sum")" | |
sum="${sum%% *}" | |
digest="$algo:$sum" | |
size="$(stat --format '%s' "$tmp/blob-data")" | |
mv "$tmp/blob-data" "$tmp/$sum" # avoid needing "mv -T" | |
mkdir -p "$oci/blobs/$algo" | |
mv -f "$tmp/$sum" "$oci/blobs/$algo/" | |
export digest size | |
jq -nc '{ digest: env.digest, size: (env.size | tonumber) }' |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment