the source code is mainly from chatGPT to support this question: https://stackoverflow.com/questions/76146859/save-a-serial-of-imagescvmat-to-a-mp4-file-in-variable-frame-rate-mode-by-us

read_image_from_mp4_imshow.cpp

#include <iostream>
#include <vector>
#include <cstring>
#include <fstream>
#include <sstream>
#include <stdexcept>
#include <opencv2/opencv.hpp>
extern "C" {
#include <libavutil/imgutils.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libswscale/swscale.h>
#include <libavutil/frame.h>
}

// below code from: normalize_ts.cpp:217:49: error: taking address of temporary array Issue #5
// joncampbell123/composite-video-simulator — https://github.com/joncampbell123/composite-video-simulator/issues/5
#ifdef av_err2str
#undef av_err2str
#include <string>
av_always_inline std::string av_err2string(int errnum) {
    char str[AV_ERROR_MAX_STRING_SIZE];
    return av_make_error_string(str, AV_ERROR_MAX_STRING_SIZE, errnum);
}
#define av_err2str(err) av_err2string(err).c_str()
#endif  // av_err2str



int main(int argc, char* argv[])
{
    // Open the input file using FFmpeg
    const char* filename = "test.mp4";
    AVFormatContext* formatContext = nullptr;
    if (avformat_open_input(&formatContext, filename, nullptr, nullptr) != 0) {
        std::cerr << "Failed to open file: " << filename << std::endl;
        return -1;
    }
    if (avformat_find_stream_info(formatContext, nullptr) < 0) {
        std::cerr << "Failed to find stream information: " << filename << std::endl;
        return -1;
    }

    // Find the first video stream and its decoder
    const AVCodec* codec = nullptr;
    int videoStreamIndex = av_find_best_stream(formatContext, AVMEDIA_TYPE_VIDEO, -1, -1, &codec, 0);
    if (videoStreamIndex < 0) {
        std::cerr << "Failed to find video stream: " << filename << std::endl;
        return -1;
    }
    AVCodecContext* codecContext = avcodec_alloc_context3(codec);
    if (!codecContext) {
        std::cerr << "Failed to allocate codec context" << std::endl;
        return -1;
    }
    if (avcodec_parameters_to_context(codecContext, formatContext->streams[videoStreamIndex]->codecpar) < 0) {
        std::cerr << "Failed to copy codec parameters to context" << std::endl;
        return -1;
    }
    if (avcodec_open2(codecContext, codec, nullptr) < 0) {
        std::cerr << "Failed to open codec" << std::endl;
        return -1;
    }

    // Allocate the frame buffer and the scaler context
    AVFrame* frame = av_frame_alloc();
    AVFrame* frameRGB = av_frame_alloc();
    uint8_t* buffer = nullptr;
    int numBytes = av_image_get_buffer_size(AV_PIX_FMT_RGB24, codecContext->width, codecContext->height, 1);
    buffer = (uint8_t*)av_malloc(numBytes * sizeof(uint8_t));
    av_image_fill_arrays(frameRGB->data, frameRGB->linesize, buffer, AV_PIX_FMT_RGB24, codecContext->width, codecContext->height, 1);
    SwsContext* swsContext = sws_getContext(codecContext->width, codecContext->height, codecContext->pix_fmt, codecContext->width, codecContext->height, AV_PIX_FMT_RGB24, SWS_BILINEAR, nullptr, nullptr, nullptr);

    // Read frames from the input file and display them using OpenCV
    AVPacket packet;
    av_init_packet(&packet);
    int frameIndex = 0;
    while (av_read_frame(formatContext, &packet) >= 0) {
        if (packet.stream_index == videoStreamIndex) {
            // Decode the frame
            int response = avcodec_send_packet(codecContext, &packet);
            if (response < 0) {
                std::cerr << "Error decoding packet: " << av_err2str(response) << std::endl;
                break;
            }
            while (response >= 0) {
                response = avcodec_receive_frame(codecContext, frame);
                if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
                    break;
                }
                else if (response < 0) {
                    std::cerr << "Error decoding frame: " << av_err2str(response) << std::endl;
                    break;
                }

                // Convert the frame to RGB and display it using OpenCV
                sws_scale(swsContext, frame->data, frame->linesize, 0, codecContext->height, frameRGB->data, frameRGB->linesize);
                cv::Mat image(codecContext->height, codecContext->width, CV_8UC3, frameRGB->data[0]);
                cv::imshow("Frame", image);

                // Get the PTS value of the frame and print it
                int64_t pts = frame->best_effort_timestamp; //av_frame_get_best_effort_timestamp(frame);
                std::cout << "Frame " << frameIndex << " PTS: " << pts << std::endl;

                // Wait for a key press, and exit if the 'q' key is pressed
                int key = cv::waitKey(1);
                if (key == 'q') {
                    break;
                }

                frameIndex++;
            }
        }
        av_packet_unref(&packet);
    }

    // Clean up resources
    avformat_close_input(&formatContext);
    avcodec_free_context(&codecContext);
    av_frame_free(&frame);
    av_frame_free(&frameRGB);
    av_free(buffer);
    sws_freeContext(swsContext);

    return 0;
}

When running the code, those messages will be shown on the console window:

Frame 0 PTS: 0
Frame 1 PTS: 4100
Frame 2 PTS: 8800
Frame 3 PTS: 12200
Frame 4 PTS: 16200
Frame 5 PTS: 22100
Frame 6 PTS: 25500
Frame 7 PTS: 30300
Frame 8 PTS: 35100
Frame 9 PTS: 40300
...

I just check the test.mp4 file with the ffprobe command, such as:

ffprobe -show_frames test.mp4 >> frames4.txt

It shows the correct content and matched content of the console, here is the frames4.txt content:

[FRAME]
media_type=video
stream_index=0
key_frame=1
pts=0
pts_time=0.000000
pkt_dts=0
pkt_dts_time=0.000000
best_effort_timestamp=0
best_effort_timestamp_time=0.000000
pkt_duration=N/A
pkt_duration_time=N/A
duration=N/A
duration_time=N/A
pkt_pos=48
pkt_size=1138
width=640
height=480
pix_fmt=yuv420p
sample_aspect_ratio=N/A
pict_type=I
coded_picture_number=0
display_picture_number=0
interlaced_frame=0
top_field_first=0
repeat_pict=0
color_range=unknown
color_space=unknown
color_primaries=unknown
color_transfer=unknown
chroma_location=left
[SIDE_DATA]
side_data_type=H.26[45] User Data Unregistered SEI message
[/SIDE_DATA]
[/FRAME]
[FRAME]
media_type=video
stream_index=0
key_frame=0
pts=4100
pts_time=0.164000
pkt_dts=4100
pkt_dts_time=0.164000
best_effort_timestamp=4100
best_effort_timestamp_time=0.164000
pkt_duration=N/A
pkt_duration_time=N/A
duration=N/A
duration_time=N/A
pkt_pos=1694
pkt_size=121
width=640
height=480
pix_fmt=yuv420p
sample_aspect_ratio=N/A
pict_type=B
coded_picture_number=3
display_picture_number=0
interlaced_frame=0
top_field_first=0
repeat_pict=0
color_range=unknown
color_space=unknown
color_primaries=unknown
color_transfer=unknown
chroma_location=left
[/FRAME]
[FRAME]
media_type=video
stream_index=0
key_frame=0
pts=8800
pts_time=0.352000
pkt_dts=8800
pkt_dts_time=0.352000
best_effort_timestamp=8800
best_effort_timestamp_time=0.352000
pkt_duration=N/A
pkt_duration_time=N/A
duration=N/A
duration_time=N/A
pkt_pos=1454
pkt_size=240
width=640
height=480
pix_fmt=yuv420p
sample_aspect_ratio=N/A
pict_type=B
coded_picture_number=2
display_picture_number=0
interlaced_frame=0
top_field_first=0
repeat_pict=0
color_range=unknown
color_space=unknown
color_primaries=unknown
color_transfer=unknown
chroma_location=left
[/FRAME]
...