Lossless JPEG sequence -> x265 and back.

alternative.rs

// An earlier attempt that is more fleshed out with some more notes.

const TRANSCODE_RESULT_EXTENSION: &str = "mkv";

impl Transcoder {
    // Transcode animation to lossless x265. x265 is 1 FPS with one frame per second.
    async fn transcode_animation(&self, frame_files: Vec<(&AnimationFrame, &Path)>) -> Result<PathBuf> {
        // ffprobe the frames to grab the pixel format
        let mut cur_pix_fmt: String = "unknown".to_owned();
        let mut first_stream: Option<FfprobeStream> = None;
        for (idx, (_, frame_file_path)) in frame_files.iter().enumerate() {
            set_current_task_status(Some(format!("Probing pixel format of frame {} of {}", idx + 1, frame_files.len())));

            let ffprobe_output = Command::new("ffprobe")
                .arg("-print_format").arg("json")
                .arg("-v").arg("quiet")
                .arg("-show_streams")
                .arg(frame_file_path)
                .stdin(Stdio::null())
                .kill_on_drop(true)
                .output()
                .await
                .with_context(|| format!("Failed to spawn ffprobe process to probe frame: {}",
                                         frame_file_path.display()))?;
            if !ffprobe_output.status.success() {
                return Err(anyhow!(
                        "ffprobe did not exit successfully, exit code: {}, probe target: {}",
                        ffprobe_output.status,
                        frame_file_path.display()
                    ))
            }
            let ffprobe_data: FfprobeData = serde_json::from_slice(&ffprobe_output.stdout)
                .with_context(|| format!(
                    "Failed to deserialize ffprobe result for frame: {}, output: {:?}",
                    frame_file_path.display(),
                    ffprobe_output
                ))?;
            let stream = ffprobe_data.streams.first()
                .context("Frame has no streams!")?;
            let input_pixel_format = stream.pix_fmt.as_str();
            if first_stream.is_none() {
                first_stream = Some(stream.clone());
            }

            if cur_pix_fmt == "unknown" {
                cur_pix_fmt = input_pixel_format.to_owned();
            } else if cur_pix_fmt != input_pixel_format {
                // Animation has frames with different color profiles, see if we can decide on a color
                // profile that is lossless for all frames
                /*let result = match input_pixel_format {
                    "yuvj444p" => match cur_pix_fmt.as_str() {
                        "yuvj420p" => Some("yuvj444p"), // Promote to YUV 4:4:4
                        "yuvj422p" => Some("yuvj444p"), // Promote to YUV 4:4:4
                        "gray" => Some("yuvj444p"), // Promote to YUV 4:4:4
                        _ => None
                    }
                    "yuvj422p" => match cur_pix_fmt.as_str() {
                        "yuvj444p" => Some("yuvj444p"), // Keep at YUV 4:4:4
                        "yuvj420p" => Some("yuvj422p"), // Promote to YUV 4:2:2
                        "gray" => Some("yuvj422p"), // Promote to YUV 4:2:2
                        _ => None
                    }
                    "yuvj420p" => match cur_pix_fmt.as_str() {
                        "yuvj444p" => Some("yuvj444p"), // Keep at YUV 4:4:4
                        "yuvj422p" => Some("yuvj422p"), // Keep at YUV 4:2:2
                        "gray" => Some("yuvj420p"), // Promote to YUV 4:2:0
                        _ => None
                    }
                    "gray" => match cur_pix_fmt.as_str() {
                        "yuvj444p" => Some("yuvj444p"), // Keep at YUV 4:4:4
                        "yuvj422p" => Some("yuvj422p"), // Keep at YUV 4:2:2
                        "yuvj420p" => Some("yuvj420p"), // Keep at YUV 4:2:0
                        "rgb24" => Some("rgb24"), // Keep at RGB
                        _ => None
                    }
                    "rgb24" => match cur_pix_fmt.as_str() {
                        "gray" => Some("rgb24"), // Promote to RGB
                        _ => None
                    }
                    _ => None
                };
                if let Some(result) = result {
                    cur_pix_fmt = result.to_owned();
                } else {
                    return Err(anyhow!("Animation contains frames with both the {cur_pix_fmt} and {input_pixel_format} color profiles. Unable to solve for a universally lossless color profile, bailing."));
                }*/

                // SCRAPPED because we can't check the integrity of the transcode with framehash
                // Give up and just copy frames
                cur_pix_fmt = "copy".to_owned();
                break;
            }
        }

        // Some pixel formats can't be encoded in HEVC if the height/width don't divide perfectly
        // Do not bother encoding these animations, instead just use copy codec
        // This algorithm is from: https://github.com/GPUOpen-LibrariesAndSDKs/AMF/blob/03e2731fa69e68ee313ddfa18afcfafac97d0387/amf/public/samples/CPPSamples/common/BitStreamParserH265.cpp#L552
        let required_divisor = match cur_pix_fmt.as_str() {
            // "yuvj400p" => (1, 1), // Doesn't exist for JPEGs
            "yuvj420p" => (2, 2),
            "yuvj422p" => (2, 1),
            "yuvj444p" => (1, 1),
            _ => (1, 1)
        };
        if let Some(first_stream) = first_stream {
            if first_stream.width % required_divisor.0 != 0 ||
                first_stream.height % required_divisor.1 != 0 {
                cur_pix_fmt = "copy".to_owned();
            }
        }

        // Note that MPV is able to play the animationa back with the correct colors, other players (VLC) shift the colors
        let output_pixel_format = match cur_pix_fmt.as_str() {
            // Lossless conversions for x265:
            "yuvj444p" => vec![
                "-c:v", "libx265",
                "-x265-params", "lossless=1",

                // yuvj444p is full-range YUV and yuv444p is limited-range YUV
                // However, we can force yuv444p to be full-range by setting the color_range
                // manually below.
                "-pix_fmt", "yuv444p",
                "-color_range", "pc", // Add tag in output container to indicate output color range is "pc" (full)
                // We tagged the output color_range as "pc", but swscaler will still attempt to convert the input colors into
                // the 'tv' color range. To prevent this, we explicitly scale the output into the 'pc' color range.
                // Also, since JPEGs are always in the "pc" color range, I am confident enough to also specify that the input
                // color range will be "pc".
                "-vf", "scale=in_range=pc:out_range=pc, fps=1"

                // Note that it is extremely painful to extract frames from this format as there is
                // no good lossless image format that supports YUV444P.
                // I tried using TIFF files but that didn't really work.
                // Perhaps we can use JPEG XL?
                // This *kinda* works but it isn't lossless:
                // ffmpeg -i animation.packed.webm -compression_algo lzw -pix_fmt yuv444p -vf 'scale=in_range=pc:out_range=tv' frames-t/out-%03d.tif
            ],
            "yuvj422p" => vec![
                "-c:v", "libx265",
                "-x265-params", "lossless=1",

                "-pix_fmt", "yuv422p",
                "-color_range", "pc",
                "-vf", "scale=in_range=pc:out_range=pc, fps=1"
            ],
            "yuvj420p" => vec![
                "-c:v", "libx265",
                "-x265-params", "lossless=1",

                "-pix_fmt", "yuv420p",
                "-color_range", "pc",
                "-vf", "scale=in_range=pc:out_range=pc, fps=1"
            ],
            "gray" => vec![
                "-c:v", "libx265",
                "-x265-params", "lossless=1",

                "-pix_fmt", "gray",
                "-color_range", "pc",
                "-vf", "scale=in_range=pc:out_range=pc, fps=1"
            ],
            "rgb24" => vec![
                "-c:v", "libx265",
                "-x265-params", "lossless=1",

                // UNTESTED!!!
                "-pix_fmt", "gbrp", // This is 8-bit RGB, just in planar form.
                "-vf", "fps=1"
            ],
            // Copy frames without transcoding
            // Also note that we don't have to dupe the last frame if we copy
            "copy" => vec![
                "-c:v", "copy",
            ],
            _ => {
                return Err(anyhow!("Animation has unknown pixel format: {}", cur_pix_fmt))
            }
        };

        // "ffmpeg -pix_fmts" lists available pixel formats
        // "ffmpeg -h encoder=libvpx-vp9" lists supported pixel formats for a specific codec

        // Confused about 'be' or 'le' codec?
        // See: https://www.reddit.com/r/ffmpeg/comments/j7egw6/difference_between_pixel_formats_ending_in_le_and/

        set_current_task_status(Some("Building concat spec".to_string()));

        // Gen concat spec path
        let (concat_spec_path, concat_spec) = self.writer.tmp_file("txt")
            .await
            .context("Failed to create temp file for concat spec.")?;

        // <BEGIN> DO NOT RETURN 1 (because we need to delete tmp concat spec)
        let transcode_result: Result<PathBuf> = async {
            // Write concat spec
            let mut concat_spec_writer = BufWriter::new(concat_spec);
            concat_spec_writer.write_all(b"ffconcat version 1.0\n").await
                .with_context(|| format!("Failed to write header to concat spec: {}", concat_spec_path.display()))?;

            let num_frames = frame_files.len();
            for (idx, (_frame, frame_path)) in frame_files.into_iter().enumerate() {
                let abs_path = tokio::fs::canonicalize(frame_path).await
                    .with_context(|| format!("Failed to canonicalize path: {}", frame_path.display()))?;
                let escaped_path = abs_path.to_str()
                    .with_context(|| format!("Frame {idx} has invalid unicode. Here is the path: {}", frame_path.display()))?
                    .replace('\\', "\\\\")
                    .replace('\'', "\\'");

                // let frame_delay_secs = (frame.delay as f64) / 1000.0;
                let frame_delay_secs = 1.0;
                concat_spec_writer.write_all(format!("file '{escaped_path}'\nduration {}\n", frame_delay_secs).as_bytes()).await
                    .with_context(|| format!("Failed to write frame {idx} to concat spec: {}", concat_spec_path.display()))?;

                // Duplicate last frame because last frame is always ignored by concat demuxer
                if cur_pix_fmt != "copy" && idx == num_frames - 1 {
                    concat_spec_writer.write_all(format!("file '{escaped_path}'\n").as_bytes()).await
                        .with_context(|| format!("Failed to write duplicate last frame to concat spec: {}", concat_spec_path.display()))?;
                }
            }

            concat_spec_writer.flush()
                .await
                .context("Failed to flush concat spec buffer!")?;
            drop(concat_spec_writer);

            set_current_task_status(Some(format!("Transcoding {num_frames} frames")));

            // Gen output path
            let output_path = self.writer.tmp_file_path(TRANSCODE_RESULT_EXTENSION)
                .await
                .context("Failed to generate temp file path!")?;

            // <BEGIN> DO NOT RETURN 2 (because we need to delete output file)
            let ffmpeg_result = async {
                let concat_args = &[
                    OsStr::new("-f"), OsStr::new("concat"),
                    OsStr::new("-safe"), OsStr::new("0"),
                    OsStr::new("-i"), concat_spec_path.as_os_str()
                ];
                // Transcode with ffmpeg
                let output = Command::new("ffmpeg")
                    .args(concat_args)
                    .args(&output_pixel_format)
                    .arg(&output_path)
                    .stdin(Stdio::null())
                    .kill_on_drop(true)
                    .output()
                    .await
                    .with_context(|| format!("Failed to spawn ffmpeg process, concat spec: {}, output path: {}",
                                             concat_spec_path.display(),
                                             output_path.display()))?;

                if !output.status.success() {
                    return Err(anyhow!(
                        "ffmpeg did not exit successfully, concat spec: {}, output path: {}, output: {:?}",
                        concat_spec_path.display(),
                        output_path.display(),
                        output
                    ))
                }

                set_current_task_status(Some("Verifying transcode is lossless".to_string()));

                // Verify transcode is lossless by comparing framehashes
                let mut framehashes_orig = read_framehashes(concat_args)
                    .await
                    .with_context(|| format!("Failed to read framehashes of raw frames: {concat_args:?}"))?;
                let transcode_res_fh_args = &[
                    OsStr::new("-i"), output_path.as_os_str()
                ];
                if !framehashes_orig.is_empty() {
                    if cur_pix_fmt != "copy" {
                        // Remove last framehash because it is a duplicate
                        framehashes_orig.remove(framehashes_orig.len() - 1);
                    }
                } else {
                    return Err(anyhow!("No framehashes found for raw frames: {concat_args:?}"));
                }
                let framehashes_new = read_framehashes(transcode_res_fh_args)
                    .await
                    .with_context(|| format!("Failed to read framehashes of transcode result: {transcode_res_fh_args:?}"))?;
                if framehashes_orig != framehashes_new {
                    return Err(anyhow!("Transcode is not lossless, framehashes mismatch. Expected: {framehashes_orig:?}. Got: {framehashes_new:?}. Ffmpeg conversion output: {output:?}"))
                }

                Ok(())
            }.await;
            // <END> DO NOT RETURN 2

            if let Err(e) = ffmpeg_result {
                // Delete output path on failure
                try_delete(&output_path).await;
                Err(e)
            } else {
                Ok(output_path)
            }
        }.await;
        // <END> DO NOT RETURN 1

        // Delete concat spec
        try_delete(&concat_spec_path).await;

        transcode_result
    }
}

async fn read_framehashes(args: &[&OsStr]) -> Result<Vec<String>> {
    let output = Command::new("ffmpeg")
        .args(args)
        .arg("-v").arg("quiet")
        .arg("-f").arg("framehash")
        .arg("-")
        .stdin(Stdio::null())
        .kill_on_drop(true)
        .output()
        .await
        .with_context(|| format!("Failed to spawn ffmpeg process, args: {args:?}"))?;

    if !output.status.success() {
        return Err(anyhow!(
                        "ffmpeg did not exit successfully, exit code: {}, args: {:?}",
                        output.status,
                        args
                    ))
    }

    let framehashes_str = String::from_utf8(output.stdout)
        .with_context(|| format!("Framehashes contain invalid UTF-8 for args: {args:?}"))?;

    Ok(framehashes_str
        .lines()
        .map(|l| l.trim())
        .filter(|l| !l.is_empty() && !l.starts_with('#'))
        .filter_map(|l| {
            l.rfind(',')
                .and_then(|idx| l.get(idx+1..)
                    .map(|h| h.trim().to_owned()))
        })
        .collect())
}

async fn try_delete(path: &Path) {
    if let Err(e) = tokio::fs::remove_file(path).await {
        warn!("Failed to delete temp file {}. Here is the full error: {:?}", path.display(), e);
    }
}

roundtrip.sh

#!/bin/sh

#Encode
magick working/INPUT.jpg -set colorspace Rec709YCbCr working/STEP1.tiff
ffmpeg -framerate 30 -i working/STEP1.tiff -c:v libx265 -x265-params lossless=1 -pix_fmt yuv444p -color_primaries bt709 -color_trc bt709 -colorspace bt709 -color_range pc -vf "setparams=color_primaries=bm9:color_trc=bt709:colorspace=bt709" working/OUT.mkv

# Decode
ffmpeg -i working/OUT.mkv -pix_fmt yuv444p -color_primaries bt709 -color_trc bt709 -colorspace bt709 -color_range pc -vf "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709" 'working/REV_STEP1_%04d.tiff'
magick working/REV_STEP1_0001.tiff -set colorspace sRGB working/REV.tiff

# STEP1.tiff should be identical to REV.tiff