FFMPEG test

benchmark_results.csv

nushell

open benchmark | from csv | update filesize_mb { $in * 100 / 941 } | rename --column { filesize_mb: filesize_ratio } | where vmaf_harmonic > 94 | where filesize_ratio < 34 | where vmaf_min >= 86 | insert vmaf_ratio {|row| (($row.vmaf_harmonic - 93) / $row.filesize_ratio) | math round --precision 2}  | sort-by vmaf_ratio filesize_ratio
╭──────┬───────┬──────────┬──────────────────┬────────────┬────────────────┬───────────┬─────────┬───────────────────┬──────────────────┬───────┬─────────────╮
│    # │  crf  │  preset  │  filesize_ratio  │ vmaf_mean  │ vmaf_harmonic  │ vmaf_min  │ qm_min  │ gemini_optimized  │ encode_time_sec  │ tune  │ vmaf_ratio  │
├──────┼───────┼──────────┼──────────────────┼────────────┼────────────────┼───────────┼─────────┼───────────────────┼──────────────────┼───────┼─────────────┤
│    0 │    24 │        5 │            27.84 │      94.35 │          94.29 │     86.37 │       0 │ false             │              316 │     0 │        0.05 │
│    1 │    24 │        3 │            28.37 │      94.54 │          94.48 │     86.39 │       0 │ false             │              750 │     0 │        0.05 │
│    2 │    24 │        4 │            28.69 │      94.46 │          94.40 │     86.67 │       0 │ false             │              408 │     0 │        0.05 │
│    3 │    24 │        5 │            31.77 │      94.51 │          94.45 │     87.33 │       4 │ false             │              346 │     0 │        0.05 │
│    4 │    24 │        3 │            32.41 │      94.68 │          94.62 │     87.34 │       4 │ false             │              819 │     0 │        0.05 │
│    5 │    21 │        5 │            33.05 │      94.83 │          94.78 │     87.40 │       0 │ false             │              304 │     0 │        0.05 │
│    6 │    24 │        4 │            33.16 │      94.63 │          94.57 │     87.57 │       4 │ false             │              416 │     0 │        0.05 │
│    7 │    21 │        5 │            27.21 │      94.76 │          94.71 │     86.06 │       0 │ false             │              304 │     1 │        0.06 │
│    8 │    21 │        4 │            27.74 │      94.79 │          94.74 │     86.17 │       0 │ false             │              413 │     1 │        0.06 │
│    9 │    24 │        2 │            28.06 │      94.74 │          94.69 │     86.53 │       0 │ false             │             1146 │     0 │        0.06 │
│   10 │    21 │        5 │            28.27 │      94.86 │          94.81 │     86.36 │       2 │ false             │              319 │     1 │        0.06 │
│   11 │    21 │        4 │            28.69 │      94.88 │          94.83 │     86.31 │       2 │ false             │              422 │     1 │        0.06 │
│   12 │    21 │        5 │            29.12 │      94.93 │          94.88 │     86.28 │       4 │ false             │              330 │     1 │        0.06 │
│   13 │    21 │        4 │            29.54 │      94.94 │          94.89 │     86.36 │       4 │ false             │              426 │     1 │        0.06 │
│   14 │    21 │        4 │            31.03 │      94.96 │          94.91 │     86.31 │ off     │ false             │              422 │     1 │        0.06 │
│   15 │    21 │        5 │            31.24 │      95.01 │          94.96 │     86.63 │ off     │ false             │              312 │     1 │        0.06 │
│   16 │    24 │        2 │            31.46 │      94.88 │          94.83 │     86.62 │       4 │ false             │             1133 │     0 │        0.06 │
│   17 │    21 │        6 │            31.46 │      94.95 │          94.90 │     86.16 │       0 │ false             │              271 │     1 │        0.06 │
│   18 │    21 │        6 │            32.94 │      95.06 │          95.01 │     86.19 │       2 │ false             │              254 │     1 │        0.06 │
│   19 │    21 │        3 │            33.37 │      94.99 │          94.94 │     87.44 │       0 │ false             │              728 │     0 │        0.06 │
│   20 │    21 │        2 │            33.48 │      95.18 │          95.13 │     87.48 │       0 │ false             │             1034 │     0 │        0.06 │
│   21 │    21 │        4 │            33.58 │      94.92 │          94.87 │     87.43 │       0 │ false             │              376 │     0 │        0.06 │
│   22 │    21 │        3 │            27.63 │      94.93 │          94.88 │     86.18 │       0 │ false             │              829 │     1 │        0.07 │
│   23 │    21 │        3 │            28.59 │      95.01 │          94.96 │     86.33 │       2 │ false             │              809 │     1 │        0.07 │
│   24 │    21 │        2 │            28.69 │      95.14 │          95.09 │     86.65 │       0 │ false             │             1349 │     1 │        0.07 │
│   25 │    21 │        3 │            29.44 │      95.06 │          95.01 │     86.59 │       4 │ false             │              782 │     1 │        0.07 │
│   26 │    21 │        2 │            29.54 │      95.20 │          95.15 │     86.85 │       2 │ false             │             1410 │     1 │        0.07 │
│   27 │    21 │        2 │            30.29 │      95.26 │          95.21 │     86.91 │       4 │ false             │             1394 │     1 │        0.07 │
│   28 │    21 │        3 │            30.82 │      95.07 │          95.02 │     86.83 │ off     │ false             │              812 │     1 │        0.07 │
│   29 │    21 │        2 │            31.88 │      95.33 │          95.28 │     87.15 │ off     │ false             │             1444 │     1 │        0.07 │
├──────┼───────┼──────────┼──────────────────┼────────────┼────────────────┼───────────┼─────────┼───────────────────┼──────────────────┼───────┼─────────────┤
│    # │  crf  │  preset  │  filesize_ratio  │ vmaf_mean  │ vmaf_harmonic  │ vmaf_min  │ qm_min  │ gemini_optimized  │ encode_time_sec  │ tune  │ vmaf_ratio  │
╰──────┴───────┴──────────┴──────────────────┴────────────┴────────────────┴───────────┴─────────┴───────────────────┴──────────────────┴───────┴─────────────╯

script.sh

#!/bin/bash

trap "echo -e '\nScript aborted by user.'; exit 1" SIGINT

if [ ! -f benchmark_results.csv ]; then
    echo "crf,preset,filesize_mb,vmaf_mean,vmaf_harmonic,vmaf_min,qm_min,gemini_optimized,encode_time_sec,tune" > benchmark_results.csv
fi

for crf in $(seq 15 3 27); do
  for preset in 2 3 4 5 6; do
		for qm_min in off 0 2 4; do
			for gemini in false; do #true; do

				echo "============================================="
				echo "Processing: CRF ${crf} | Preset ${preset} | QM ${qm_min} | Gemini ${gemini} "
				echo "============================================="

				OUTPUT_FILE="${crf}-${preset}-${qm_min}-${gemini}-tune1.mkv"
				JSON_LOG="${crf}-${preset}-${qm_min}-${gemini}-tune1_vmaf.json"

				if [ -e "$OUTPUT_FILE" ]; then continue; fi
				
				# -svtav1-params keyint=10s:tune=0:enable-overlays=1:scd=1:lookahead=120:enable-variance-boost=1 \
				# SVTPARAMS="keyint=10s:tune=0:enable-overlays=1:scd=1:lookahead=120:enable-variance-boost=1:tf-strength=1:enable-qm=1:qm-min=${qm_min}:qm-max=12:sharpness=1"
				SVTPARAMS="keyint=10s:tune=1:enable-overlays=1:scd=1:lookahead=120:enable-variance-boost=1:tf-strength=1:sharpness=1:sharpness=1"
				if [ "$qm_min" != "off" ]; then
				    SVTPARAMS="${SVTPARAMS}:enable-qm=1:qm-min=${qm_min}"
				fi
				if [ "$gemini" = "true" ]; then
				    SVTPARAMS="${SVTPARAMS}:aq-mode=1:cb-qpoff=1:cr-qpoff=1"
				fi

				start_time=$SECONDS

				docker run --rm --init -v $(pwd):/config linuxserver/ffmpeg \
					-i /config/base.mkv \
					-hide_banner \
					-crf $crf \
					-preset $preset \
					-c:v libsvtav1 \
					-svtav1-params "${SVTPARAMS}" \
					-pix_fmt yuv420p10le \
					/config/"$OUTPUT_FILE"

				encode_time=$(( SECONDS - start_time ))

				docker run --rm --init -v $(pwd):/config linuxserver/ffmpeg \
					-hide_banner \
					-i /config/"$OUTPUT_FILE" \
					-i /config/base.mkv \
					-filter_complex "[0:v][1:v]scale=rw:rh[distorted];[distorted][1:v]libvmaf=model=version=vmaf_v0.6.1neg:log_fmt=json:log_path=/config/$JSON_LOG" \
					-f null -

				filesize=$(du -m "$OUTPUT_FILE" | cut -f1)

				vmaf_mean=$(jq '.pooled_metrics.vmaf.mean' "$JSON_LOG")
				vmaf_harmonic=$(jq '.pooled_metrics.vmaf.harmonic_mean' "$JSON_LOG")
				vmaf_min=$(jq '.pooled_metrics.vmaf.min' "$JSON_LOG")

				echo "${crf},${preset},${filesize},${vmaf_mean},${vmaf_harmonic},${vmaf_min},${qm_min},${gemini},${encode_time},1" >> benchmark_results.csv
			done
		done
  done
done

The VMAF ratio insert vmaf_ratio {|row| (($row.vmaf_harmonic - 93) / $row.filesize_ratio) | math round --precision 2} changes a lot depending on the offset you choose, it might be worthwhile changing it depending on what you value. Overall I quite like the very last entry, giving a vmaf score of 95.28 while staying below 33% original file size. The values correspond to

docker run --rm --init -v $(pwd):/config linuxserver/ffmpeg \
	-i /config/base.mkv \
	-hide_banner \
	-crf 21 \
	-preset 2 \
	-c:v libsvtav1 \
	-svtav1-params "keyint=10s:tune=1:enable-overlays=1:scd=1:lookahead=120:enable-variance-boost=1:tf-strength=1:sharpness=1" \
	-pix_fmt yuv420p10le \
	/config/"$OUTPUT_FILE"