Created
March 16, 2026 11:59
-
-
Save v-fox/df4b13c9988a9055c4fe4e9449282730 to your computer and use it in GitHub Desktop.
on-GPU motion interpolation playback in mpv
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/python | |
| # vim: set ft=python: | |
| # this should be used via ~/.config/mpv/mpv.conf: | |
| # [default] | |
| # profile=HQ-MI-RIFE | |
| # | |
| # [HQ-MI-RIFE] | |
| # profile-desc="gpu-hq profile with RIFE interpolation via VS, assuming HDR display under Wayland" | |
| # profile=gpu-hq | |
| # vo=gpu-next,gpu,dmabuf-wayland, | |
| # gpu-api=vulkan,auto | |
| # vulkan-swap-mode=fifo-relaxed | |
| # swapchain-depth=4 | |
| # vulkan-queue-count=4 | |
| # vulkan-async-transfer=yes | |
| # vulkan-async-compute=yes | |
| # hwdec=vulkan-copy,vaapi-copy,no | |
| # blend-subtitles=no | |
| # dither-depth=10 | |
| # # "error-diffusion" or "ordered"; ED is heavy but nice | |
| # #dither=ordered | |
| # dither-size-fruit=8 | |
| # error-diffusion=sierra-2 | |
| # hdr-compute-peak=yes | |
| # allow-delayed-peak-detect=yes | |
| # hdr-peak-percentile=92 | |
| # hdr-contrast-recovery=0.8 | |
| # hdr-contrast-smoothness=2.4 | |
| # tone-mapping-max-boost=1.12 | |
| # # bastardize SDR to "stretch" it into HDR colorspace? this breaks standards but looks way better | |
| # inverse-tone-mapping=yes | |
| # # 'perceptual', 'relative' and 'linear' are sane options | |
| # gamut-mapping-mode=perceptual | |
| # target-colorspace-hint=auto | |
| # target-colorspace-hint-mode=source-dynamic | |
| # target-prim=bt.2020 | |
| # # use "dci-p3" if your display is close to it, remove if your display is true bt.2020 | |
| # target-gamut=dci-p3 | |
| # # "pq" is "HDR10+", the only dynamic mode supported by Wayland right now | |
| # target-trc=pq | |
| # # "inf" if display is OLED or does auto-correction (which is likely in "HDR10+" mode) | |
| # target-contrast=inf | |
| # # this MUST match peak manually calibrated in your Wayland compositor and display is likely does auto-correction (hiding real number, so you cannot just assume) | |
| # target-peak=1000 | |
| # # expected default is 203 but you may want to use 160 or even 120 (also doing the same in Wayland for all SDR content) | |
| # #image-subs-hdr-peak=160 | |
| # #sub-hdr-peak=160 | |
| # icc-profile-auto=yes | |
| # icc-use-luma=yes | |
| # # mpv's built-in interpolation is a blurry mess but it may smooth out some jitter as a post-effect | |
| # interpolation=yes | |
| # video-sync=display-resample-vdrop | |
| # framedrop=vo | |
| # hr-seek-framedrop=no | |
| # #hr-seek=no | |
| # interpolation-threshold=0.1 | |
| # # "display-vdrop" may drop more but be more stable than "display-resample-vdrop" | |
| # video-sync-max-video-change=5 | |
| # video-sync-max-audio-change=0.075 | |
| # tscale=sphinx | |
| # should be put into ${HOME}/.config/mpv/vapoursynth/ | |
| # vf-add=vapoursynth=~~/vapoursynth/MotionInterpolation-VSNCNN-RIFE.vpy:2:4 | |
| import vapoursynth as vs | |
| import functools | |
| # alternative to core.resize.Bicubic | |
| #import mvsfunc | |
| # https://www.vapoursynth.com/2021/09/r55-audio-support-and-improved-performance/ | |
| core = vs.core | |
| GPU = 0 ## GPU index used, 0 is the first in order | |
| GPU_THREADS = 4 ## <1|2|3> Number of GPU threads | |
| GPU_FP16 = 0 ## <0|1> Allow FP16 format? likely to be broken | |
| GPU_TILESIZE = 128 ## <32|64|128> Side-length of square tiles in pixels: v4.25_lite = 128, else v4.25 & v4.26 = 64, all else = 32 | |
| STRETCHED = False ## <False|True> force size divisible by GPU_TILESIZE? this should be done transparently by the interpolation function | |
| COLORSPACE = "HDR" ## <"SD"|"HD"|"HDR"> template for DVD/SDR (bt601), BD/SDR (bt709) and HDR (bt2020/st2084=PQ) | |
| METADATA_FIX = 0 ## hardcode basic metadata to force RGB->YUV post-conversion? it should work without it | |
| OUT_PLANES = vs.YUV444P16 ## <vs.YUV420P8|vs.YUV422P10|vs.YUV444P16> format to feed back into mpv in the most moronic syntax: as VS'es constant that mixes context, planes and bitness | |
| OUT_BITS = 16 ## <8|10|12|16|32> bits in the format to feed mpv back, should be the same as above | |
| # RIFE/NCNN | |
| # https://github.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan/blob/c3ec6aabc07c8fa37a4f58d7fed9e2ad1fc1b13f/RIFE/plugin.cpp#L287 | |
| # good values are v1-UHD=2, v3.9-fast=9, v4.4-fast=19, v4.24-ensemble=68, v4.25-lite=70, v4.26|.25-heavy|.26-large=72|71|73 | |
| RIFE_MODEL = 70 | |
| TTA = 0 ## <0|1> Some crap only for ancient models v1|2 (numbers 0-10) | |
| UHD = 1 ## <0|1> Allow optimizations for 4K? | |
| SCD = 1 ## <0|1|2> Scene change detection mode, 0 to disable | |
| SCD_STATIC = 1 ## <0|1> Detect static scenes to skip using VMAF plugin? | |
| # MLRT/NCNN | |
| # see `ls -l /usr/lib64/vapoursynth/models/rife*/*` | |
| # good options are "rife/rife_v4.25_lite.onnx" | |
| MLRT_PATH = "/usr/lib64/vapoursynth/models/rife_v2/rife_v4.25_lite.onnx" | |
| FLEXIBLE = "" ## <""|"True"> "flexible output"? | |
| # MLRT/non-NCNN | |
| # v4_0 = 40, v4_2 = 42, v4_3 = 43, v4_4 = 44, v4_5 = 45, v4_6 = 46, v4_7 = 47, v4_8 = 48, v4_9 = 49, v4_10 = 410, v4_11 = 411, v4_12 = 412, v4_12_lite = 4121, v4_13 = 413, v4_13_lite = 4131, v4_14 = 414, v4_14_lite = 4141, v4_15 = 415, v4_15_lite = 4151, v4_16_lite = 4161, v4_17 = 417, v4_17_lite = 4171, v4_18 = 418, v4_19 = 419, v4_20 = 420, v4_21 = 421, v4_22 = 422, v4_22_lite = 4221, v4_23 = 423, v4_24 = 424, v4_25 = 425, v4_25_lite = 4251, v4_25_heavy = 4252, v4_26 = 426, v4_26_heavy = 4262 | |
| MLRT_RIFE_MODEL = 4251 | |
| ABOVE2K = True ## <True|False> Whether to interpolate frames for sources exceeding DCI2K resolution | |
| EXPAD = False ## <True|False> Whether to use external padding/cropping processing | |
| STATIC = False ## <True|False> Whether to use static engine (requires preprocessing for different resolutions); dynamic engine adapts to different resolutions (QVGA→DCI4K) | |
| VRAM = 6144 ## Integer, constrain VRAM (MiB), minimum for static engine is 128 (dynamic engine automatically doubles), set below this value to maximize utilization | |
| # dithering modes are 0=ordered, 1=closest round, 2=round, 3=Sierra-2-4A ED (fast), 4=Stucki ED (good edges, bad gradients), 5=Atkinson ED (visible patterns), 6=Floyd-Steinberg ED, 7=Ostromoukhov ED (slow and integer-only), 8=Void and cluster halftone | |
| # dithering pattern can be 4/8/16/32 wide; rotation may not be good for slow displays, like VA | |
| DITH_TYPE = "error_diffusion" ## <"error_diffusion"|"ordered"> ED is way better but heavier | |
| DITH_MODE = 9 | |
| DITH_ROTATE = 0 | |
| DITH_STATIC_NOISE = 0 | |
| DITH_AMP_ORDERED = 2 | |
| DITH_AMP_NOISE = 0 | |
| DITH_PATTERN = 32 | |
| max_mpix = 10.5 | |
| max_mpix_hq = 0.922 | |
| # the bigger the source's framerate, the more often frame analyzation should happen, so CPU load rises exponentially | |
| # especially if software realtime format conversion is used in the player | |
| min_src_fps = 9 | |
| max_src_fps_lq = 31 | |
| max_src_fps_hq = 51 | |
| # maximum allowed target frame-rate for interpolation | |
| max_dst_fps = 60 | |
| # ideally, this should have the same precision as what GPU driver uses for driving the display but ≥8 gives out errors | |
| # here's it's an actual value of monitor's refresh rate | |
| dst_fps = float(display_fps) | |
| dst_fps_num = int(dst_fps * 1e6) | |
| dst_fps_den = int(1e6) | |
| PLAYER_FAIL = False | |
| if "video_in" in globals(): | |
| # realtime | |
| clip = video_in | |
| # mpv may screw up and report 0 or something random as container_fps | |
| if container_fps == 0: | |
| print("[rife-info]: player screwed up in determining video's fps, so we skip all processing !") | |
| PLAYER_FAIL = True | |
| else: | |
| # Needed because clip FPS is missing | |
| print("[rife-info]: Container fps reported by mpv is {}".format(container_fps)) | |
| # how precise this should be ? definitively, not less than e4 but is bigger necessary ? | |
| src_fps_num = int(container_fps * 1e8) | |
| src_fps_den = int(1e8) | |
| clip = core.std.AssumeFPS(clip, fpsnum = src_fps_num, fpsden = src_fps_den) | |
| print("[rife-info]: Running in realtime with {} / {} initial src / dst fps".format(clip.fps_num / clip.fps_den, dst_fps_num / dst_fps_den)) | |
| else: | |
| # run with vspipe | |
| clip = core.ffms2.Source(source=in_filename) | |
| print("[rife-info]: Running with vspipe with {} / {} initial src / dst fps".format(clip.fps_num / clip.fps_den, dst_fps_num / dst_fps_den)) | |
| if not PLAYER_FAIL: | |
| initial_width = clip.width | |
| initial_height = clip.height | |
| # resolution in megapixels. 1080p ≈ 2MP, 720p ≈ 1MP | |
| mpix = initial_width * initial_height / 1000000 | |
| src_fps = clip.fps_num / clip.fps_den | |
| else: | |
| mpix = 999 | |
| src_fps = 999 | |
| # Skip interpolation for >max_mpix megapixel or =>max_src_fps Hz content due to performance | |
| if mpix > max_mpix_hq: | |
| max_src_fps = max_src_fps_lq | |
| else: | |
| max_src_fps = max_src_fps_hq | |
| if not (PLAYER_FAIL or \ | |
| (mpix > max_mpix) or \ | |
| (src_fps < min_src_fps) or \ | |
| (src_fps > max_src_fps) or \ | |
| (src_fps >= max_dst_fps)): | |
| # info about video | |
| #print(clip.get_frame(0).props) | |
| print("[rife-info]: video size is {}x{} which is {} megapixels".format(clip.width, clip.height, mpix)) | |
| # Pre-processing this garbage | |
| print("[rife-info]: detecting movement") | |
| clip = core.misc.SCDetect(clip) | |
| # this needs to be checked in VS's guts | |
| COLOR_709 = 1 | |
| COLOR_2020ncl = 5 | |
| COLOR_170m = 6 | |
| if COLORSPACE == "HDR": | |
| COLOR_MATRIX = COLOR_2020ncl | |
| COLOR_MATRIX_S = "2020ncl" | |
| COLOR_TRANSFER = "st2084" | |
| COLOR_PRIMARIES = "2020" | |
| elif COLORSPACE == "HD": | |
| COLOR_MATRIX = COLOR_709 | |
| COLOR_MATRIX_S = "709" | |
| COLOR_TRANSFER = "709" | |
| COLOR_PRIMARIES = "709" | |
| else: | |
| COLOR_MATRIX = COLOR_170m | |
| COLOR_MATRIX_S = "170m" | |
| COLOR_TRANSFER = "601" | |
| COLOR_PRIMARIES = "709" | |
| if STRETCHED: | |
| # make sure that dimensions are divisible by GPU_TILESIZE | |
| target_width = ((initial_width + (GPU_TILESIZE - 1)) // GPU_TILESIZE) * GPU_TILESIZE | |
| target_height = ((initial_height + (GPU_TILESIZE - 1)) // GPU_TILESIZE) * GPU_TILESIZE | |
| else: | |
| target_width = initial_width | |
| target_height = initial_height | |
| if GPU_FP16: | |
| BITS = 16 | |
| print("[rife-format]: upsampling to 16 bits at {}x{}".format(target_width, target_height)) | |
| clip = core.resize.Bicubic(clip, format=vs.RGBH, width=target_width, height=target_height, dither_type=DITH_TYPE, matrix_in_s=COLOR_MATRIX_S, \ | |
| transfer_s=COLOR_TRANSFER, primaries_s=COLOR_PRIMARIES, \ | |
| transfer_in_s=COLOR_TRANSFER, primaries_in_s=COLOR_PRIMARIES) | |
| else: | |
| BITS = 32 | |
| print("[rife-format]: upsampling to 32 bits at {}x{}".format(target_width, target_height)) | |
| clip = core.resize.Bicubic(clip, format=vs.RGBS, width=target_width, height=target_height, dither_type=DITH_TYPE, matrix_in_s=COLOR_MATRIX_S, \ | |
| transfer_s=COLOR_TRANSFER, primaries_s=COLOR_PRIMARIES, \ | |
| transfer_in_s=COLOR_TRANSFER, primaries_in_s=COLOR_PRIMARIES) | |
| #clip = core.fmtc.bitdepth(clip=clip, flt=1, bits=32, \ | |
| # dmode=DITH_MODE, dyn=DITH_ROTATE, staticnoise=DITH_STATIC_NOISE,\ | |
| # ampo=DITH_AMP_ORDERED, ampn=DITH_AMP_NOISE, patsize=DITH_PATTERN) | |
| # alternative | |
| #clip = mvsfunc.ToRGB(clip, depth=32, matrix=COLOR_MATRIX_S) | |
| # The actual interpolation | |
| print("[rife-interpolation]: using model {} with {} threads on GPU #{}, at {} framerate".format(RIFE_MODEL, GPU_THREADS, GPU, (dst_fps_num/dst_fps_den))) | |
| # https://github.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan/blob/c3ec6aabc07c8fa37a4f58d7fed9e2ad1fc1b13f/RIFE/plugin.cpp#L669 | |
| if RIFE_MODEL > 10: | |
| clip = core.rife.RIFE(clip, model=RIFE_MODEL, fps_num=dst_fps_num, fps_den=dst_fps_den, \ | |
| uhd=UHD, sc=SCD, skip=SCD_STATIC, gpu_id=GPU, gpu_thread=GPU_THREADS) | |
| else: | |
| clip = core.rife.RIFE(clip, model=RIFE_MODEL, \ | |
| tta=TTA, uhd=UHD, sc=SCD, skip=SCD_STATIC, gpu_id=GPU, gpu_thread=GPU_THREADS) | |
| #print("[rife-interpolation]: using {} with GPU {}, {} threads and {} tile-size".format(RIFE_PATH, GPU, GPU_THREADS, GPU_TILESIZE)) | |
| #clip = core.ncnn.Model(clips=clip, network_path=MLRT_PATH, flexible_output_prop=FLEXIBLE, device_id=GPU, num_streams=GPU_THREADS, tilesize=GPU_TILESIZE, fp16=GPU_FP16, output_format=GPU_FP16) | |
| #clip = core.mlrt.RIFE(clip=clip, model=MLRT_RIFE_MODEL, fps_in=src_fps_num, fps_num=dst_fps_num, fps_den=dst_fps_den, \ | |
| # lt_d2k=ABOVE2K, ext_proc=EXPAD, t_tta=ENSEMBLE, sc_mode=SCD, gpu=GPU, gpu_t=GPU_THREADS, st_eng=STATIC, ws_size=VRAM) | |
| # hardcode conversion hints for this crap? this might break real metadata, if it's not already broken | |
| if METADATA_FIX: | |
| print("[rife-format]: hardcoding basic frame metadata to massage the dumb conversion function") | |
| # 0 = unspecified (RGB has no matrix) | |
| clip = core.std.SetFrameProp(clip, prop="_MatrixCoefficients", intval=0) | |
| # 9 = BT.2020; 1 = BT.709 | |
| if COLORSPACE == "HDR": | |
| clip = core.std.SetFrameProp(clip, prop="_ColourPrimaries", intval=9) | |
| # 16 = PQ (ST.2084) | |
| clip = core.std.SetFrameProp(clip, prop="_TransferCharacteristics", intval=16) | |
| else: | |
| clip = core.std.SetFrameProp(clip, prop="_ColourPrimaries", intval=1) | |
| print("[rife-format]: converting back into a dumber format: {}-bit YUV with {}/{}/{} matrix/transfer/primaries".format(OUT_BITS, COLOR_MATRIX_S, COLOR_TRANSFER, COLOR_PRIMARIES)) | |
| # "matrix=COLOR_MATRIX, matrix_in_s=COLOR_MATRIX_S, transfer_in_s=COLOR_TRANSFER, primaries_in_s=COLOR_PRIMARIES," | |
| clip = core.resize.Bicubic(clip, format=OUT_PLANES, dither_type=DITH_TYPE, matrix_s=COLOR_MATRIX_S) | |
| #clip = core.fmtc.bitdepth(clip, flt=0, bits=OUT_BITS, \ | |
| # dmode=DITH_MODE, dyn=DITH_ROTATE, staticnoise=DITH_STATIC_NOISE,\ | |
| # ampo=DITH_AMP_ORDERED, ampn=DITH_AMP_NOISE, patsize=DITH_PATTERN) | |
| clip.set_output() | |
| else: | |
| if PLAYER_FAIL: | |
| print("[rife-bail]: Skipping motion interpolation due to player's stupidity in determining video's framerate !!!") | |
| elif mpix > max_mpix: | |
| print("[rife-bail]: Skipping motion interpolation for {} megapixel video that surpasses limit of {} !!!".format(mpix,max_mpix)) | |
| elif src_fps < min_src_fps: | |
| print("[rife-bail]: Skipping motion interpolation for {} Hz video that's slower than minimum limit of {} !!!".format(src_fps,min_src_fps)) | |
| elif src_fps > max_src_fps: | |
| print("[rife-bail]: Skipping motion interpolation for {} Hz video that surpasses maximum limit of {} !!!".format(src_fps,max_src_fps)) | |
| elif src_fps >= max_dst_fps: | |
| print("[rife-bail]: Skipping meaningless {}->{} interpolation !!!".format(src_fps,max_dst_fps)) | |
| else: | |
| print("[rife-bail]: Skipping motion interpolation for some reason…") | |
| clip = video_in | |
| clip.set_output() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment