ariG23498

ariG23498

#!/usr/bin/env bash
set -euo pipefail

printf '%-10s %-9s %8s %10s %10s %10s\n' mode precision tokens tok/s seconds peak_GB
printf '%-10s %-9s %8s %10s %10s %10s\n' ---------- --------- -------- ---------- ---------- ----------
uv run example.py "$@" 2>/dev/null | tail -1
uv run example.py --nvfp4 "$@" 2>/dev/null | tail -1
uv run example.py --compile "$@" 2>/dev/null | tail -1
uv run example.py --nvfp4 --compile "$@" 2>/dev/null | tail -1

ariG23498

import ast
import json
import re

import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle
from transformers import AutoProcessor, AutoModelForMultimodalLM
from transformers.image_utils import load_image

ariG23498

import torch
import torch.nn as nn
import torch.nn.functional as F
from liger_kernel.transformers import LigerTiledGEGLUMLP
from kernels import get_kernel

# create the inputs and the conigurations
device = "cuda"
batch = 128
seq = 16

ariG23498

import torch
import spaces
from transformers import pipeline

# init pipeline
MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
pipe = pipeline(
  task="text-generation",
  model=MODEL_ID,
  use_cache=False # Note: This is important to bypass the `DynamicCache` not being registerd issue 

ariG23498

from huggingface_hub import hf_hub_download
import tarfile

dataset_id = "sayakpaul/ucf101-subset"
fname = "UCF101_subset.tar.gz"
fpath = hf_hub_download(repo_id=dataset_id, filename=fname, repo_type="dataset")

with tarfile.open(fpath) as t:
    t.extractall(".")

ariG23498

!pip install -Uq transformers bitsandbytes

import torch
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
)

model_name = "ariG23498/QwQ-32B-nf4"
tokenizer = AutoTokenizer.from_pretrained(model_name)

ariG23498

# Taken from: https://gist.github.com/sayakpaul/23862a2e7f5ab73dfdcc513751289bea

from diffusers import FluxPipeline, FluxTransformer2DModel
from transformers import T5EncoderModel
import torch
import gc


def flush():
    gc.collect()

ariG23498

ariG23498