ddh0

Tensor Type Testing

Tip

Skip to the bottom of this document for a TL;DR

For more info, see llama.cpp #12511: Handle user-defined quantization levels for additional tensors by @EAddario

Testing done by @ddh0 using this branch as of committ 5a304b8. Using libllama built for Linux CUDA.

Quantization naming scheme

ddh0

import os
import torch
import psutil
import datasets
import glob

from transformers import (
    AutoTokenizer, LlamaConfig, LlamaForCausalLM, Trainer, TrainingArguments,
    DataCollatorForLanguageModeling
)

ddh0

# hfget.py
# Python 3.11.2

# Hugging Face Filesystem API:
# https://huggingface.co/docs/huggingface_hub/en/guides/hf_file_system

import os
import sys
import time
from typing import Union

ddh0

import easy_llama as ez
from subprocess import run
USER = ez.utils.USER_STYLE
BOT = ez.utils.BOT_STYLE
DIM = ez.utils.DIM_STYLE
RESET = ez.utils.RESET_ALL
def say(something: str) -> None:
    run(['say', something])
Llama3 = ez.Model(
    'Meta-Llama-3-8B-Instruct-q8_0.gguf',

ddh0

# safetensors_convert_fp16_to_8bit.py
# Python 3.11.7

import safetensors.torch
import safetensors
import torch
import os

# blacklist takes priority over whitelist
# a tensor will only be cast if it matches the whitelist but not the blacklist