FRIDA GGUF тестирование

evaluation.frida-gguf.ipynb

{
 "cells": [
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "from encodechka_eval import tasks\n",
    "# from encodechka_eval.bert_embedders import embed_bert_both, get_word_vectors_with_bert\n",
    "from openai import OpenAI\n",
    "import numpy as np"
   ],
   "id": "e67cbe9a",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "24e5e3d9",
   "metadata": {},
   "source": [
    "from importlib import reload\n",
    "\n",
    "reload(tasks)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "99873d77",
   "metadata": {},
   "source": [
    "import os\n",
    "\n",
    "DATA_PATH_NAME = 'ENCODECHKA_DATA_PATH'\n",
    "os.environ[DATA_PATH_NAME] = 'encodechka_eval'"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "m = model = tokenizer = 'frida'\n",
    "\n",
    "client_cls = OpenAI(\n",
    "    api_key=\"~\",\n",
    "    base_url=\"http://gpu02:8888/v1\"\n",
    ")\n",
    "\n",
    "client_mean = OpenAI(\n",
    "    api_key=\"~\",\n",
    "    base_url=\"http://gpu02:9999/v1\"\n",
    ")\n",
    "\n",
    "\n",
    "def normalize(v):\n",
    "    v = np.array(v)\n",
    "    return v / (np.linalg.norm(v) + 1e-9)\n",
    "\n",
    "\n",
    "def embed_bert_both(text, model, tokenizer, max_length=128):\n",
    "    response_cls = client_cls.embeddings.create(\n",
    "        model=model,\n",
    "        input=text,\n",
    "    )\n",
    "    response_mean = client_mean.embeddings.create(\n",
    "        model=model,\n",
    "        input=text,\n",
    "    )\n",
    "    cls = np.array(response_cls.data[0].embedding)\n",
    "    mean = np.array(response_mean.data[0].embedding)\n",
    "    cls_norm = normalize(cls)\n",
    "    mean_norm = normalize(mean)\n",
    "\n",
    "    return {\n",
    "        'cls':       cls,\n",
    "        'mean':      mean,\n",
    "        'cls_norm':  cls_norm,\n",
    "        'mean_norm': mean_norm\n",
    "    }\n",
    "\n",
    "\n",
    "def get_word_vectors_with_bert(words, model, tokenizer, return_raw=False):\n",
    "    def normalize(v):\n",
    "        v = np.array(v)\n",
    "        return v / (np.linalg.norm(v) + 1e-9)\n",
    "\n",
    "    try:\n",
    "        response = client_cls.embeddings.create(\n",
    "            model=model,\n",
    "            input=words,\n",
    "        )\n",
    "    except Exception as e:\n",
    "        print(f\"[get_word_vectors_with_bert] Error during embedding request: {e}\")\n",
    "        return {i: np.zeros(768) for i in range(len(words))}\n",
    "\n",
    "    embeddings = [normalize(obj.embedding) for obj in response.data]\n",
    "\n",
    "    if return_raw:\n",
    "        return embeddings, words\n",
    "\n",
    "    id2vecs = {i: emb for i, emb in enumerate(embeddings)}\n",
    "    return id2vecs"
   ],
   "id": "b3f1edfda54a3117",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "62f9b88a",
   "metadata": {},
   "source": [
    "stsb_task = tasks.STSBTask()\n",
    "stsb_task.eval(lambda x: embed_bert_both(x, model, tokenizer), m)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "199b2bdc",
   "metadata": {},
   "source": [
    "para_task = tasks.ParaphraserTask()\n",
    "para_task.eval(lambda x: embed_bert_both(x, model, tokenizer), m)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "1a81480b",
   "metadata": {},
   "source": [
    "xnli_task = tasks.XnliTask()\n",
    "xnli_task.eval(lambda x: embed_bert_both(x, model, tokenizer), m)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "a3756015",
   "metadata": {},
   "source": [
    "senti_task = tasks.SentimentTask()\n",
    "senti_task.eval(lambda x: embed_bert_both(x, model, tokenizer), m)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "e1d5dd24",
   "metadata": {},
   "source": [
    "tox_task = tasks.ToxicityTask()\n",
    "tox_task.eval(lambda x: embed_bert_both(x, model, tokenizer), m)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "ad0f9609",
   "metadata": {},
   "source": [
    "inap_task = tasks.InappropriatenessTask()\n",
    "inap_task.eval(lambda x: embed_bert_both(x, model, tokenizer), m)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "244f2c82",
   "metadata": {},
   "source": [
    "intents_task = tasks.IntentsTask()\n",
    "intents_task.eval(lambda x: embed_bert_both(x, model, tokenizer), m)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "106c4e3f",
   "metadata": {},
   "source": [
    "intentsx_task = tasks.IntentsXTask()\n",
    "intentsx_task.eval(lambda x: embed_bert_both(x, model, tokenizer), m)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "529ca496",
   "metadata": {},
   "source": [
    "factru_task = tasks.FactRuTask()\n",
    "factru_task.eval(lambda words: get_word_vectors_with_bert(words, model=model, tokenizer=tokenizer), m)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "36d5993a",
   "metadata": {},
   "source": [
    "rudr_task = tasks.RudrTask()\n",
    "rudr_task.eval(lambda words: get_word_vectors_with_bert(words, model=model, tokenizer=tokenizer), m)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "df527b39",
   "metadata": {},
   "source": [
    "speed_task_cpu = tasks.SpeedTask()\n",
    "speed_task_cpu.eval(lambda x: embed_bert_both(x, model, tokenizer), m)"
   ],
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}

results.md

Name	Original	GGUF f32	GGUF f16	GGUF q8_0	GGUF tq2_0	GGUF tq1_0
STSBTask	0.8444961710957753	0.7736137897844315	0.7736028484956837	0.773361958282741	0.3647773184116074	0.3647773184116074
ParaphraserTask	0.760147408288547	0.5760219715717907	0.5760429923959474	0.5761467826017955	0.11769859613134119	0.11769859613134119
XnliTask	0.4796407185628742	0.4165668662674651	0.4165668662674651	0.4155688622754491	0.3393213572854291	0.3393213572854291
SentimentTask	0.836	0.7983333333333333	0.7976666666666666	0.799	0.5176666666666667	0.5176666666666667
ToxicityTask	0.989866	0.982673	0.98267	0.982684	0.724437	0.724437
InappropriatenessTask	0.8463448532935726	0.8284147185686138	0.8285707439160764	0.828561158741147	0.5766776718675115	0.5766776718675115
IntentsTask	0.8034	0.7386	0.7386	0.7392	0.2868	0.2868
IntentsXTask	0.781	0.6956	0.6956	0.6942	0.1096	0.1096
FactRuTask	0.24240861548860063	0.12707425468979844	0.12693111870225876	0.12664414521133774	0.05964525242263313	0.05964525242263313
RudrTask	0.26938953157907136	0.15240762044883197	0.15240762044883197	0.1553384424039951	0.060060972146900204	0.060060972146900204
SpeedTask (gpu)	28.902501265207924	-	-	-	-	-
SpeedTask (cpu)	994.7337913513184	184.08727248509723	110.00351905822754	81.11035426457723	53.92512798309326	90.35333156585693