My local run of Deep Floyd (XL) model

IF-I-XL-v1.0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "63b46f45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version is 2.0.0+cu117\n",
      "Use Xformers: False\n"
     ]
    }
   ],
   "source": [
    "from packaging import version\n",
    "import torch\n",
    "tv = torch.__version__\n",
    "print(\"PyTorch version is %s\" % tv)\n",
    "useXformers = True\n",
    "if version.parse(tv) >= version.parse(\"2.0.0\"):\n",
    "    useXformers = False\n",
    "    \n",
    "print(\"Use Xformers: %s\" % useXformers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "501089af",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "import torch\n",
    "\n",
    "def flush():\n",
    "  gc.collect()\n",
    "  torch.cuda.empty_cache()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "35c862b7-f586-430e-9e75-c40d77db41fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A matching Triton is not available, some optimizations will not be enabled.\n",
      "Error caught was: No module named 'triton'\n",
      "watermarker\\diffusion_pytorch_model.safetensors not found\n",
      "\n",
      "A mixture of fp16 and non-fp16 filenames will be loaded.\n",
      "Loaded fp16 filenames:\n",
      "[text_encoder/pytorch_model.fp16-00002-of-00002.bin, text_encoder/pytorch_model.fp16-00001-of-00002.bin, unet/diffusion_pytorch_model.fp16.bin, safety_checker/pytorch_model.fp16.bin]\n",
      "Loaded non-fp16 filenames:\n",
      "[watermarker/diffusion_pytorch_model.bin\n",
      "If this behavior is not expected, please check your folder structure.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a635faf03d08457ca41ba50648196870",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import transformers\n",
    "from diffusers import DiffusionPipeline\n",
    "from diffusers.utils import pt_to_pil\n",
    "\n",
    "# stage 1\n",
    "stage_1 = DiffusionPipeline.from_pretrained(\"DeepFloyd/IF-I-XL-v1.0\", variant=\"fp16\", torch_dtype=torch.float16)\n",
    "if useXformers:\n",
    "    stage_1.enable_xformers_memory_efficient_attention()\n",
    "stage_1.enable_model_cpu_offload()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b9964689-811a-40a1-a531-d65cc0790539",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "watermarker\\diffusion_pytorch_model.safetensors not found\n",
      "\n",
      "A mixture of fp16 and non-fp16 filenames will be loaded.\n",
      "Loaded fp16 filenames:\n",
      "[text_encoder/pytorch_model.fp16-00002-of-00002.bin, text_encoder/pytorch_model.fp16-00001-of-00002.bin, unet/diffusion_pytorch_model.fp16.bin, safety_checker/pytorch_model.fp16.bin]\n",
      "Loaded non-fp16 filenames:\n",
      "[watermarker/diffusion_pytorch_model.bin\n",
      "If this behavior is not expected, please check your folder structure.\n",
      "`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config[\"id2label\"]` will be overriden.\n"
     ]
    }
   ],
   "source": [
    "# stage 2\n",
    "stage_2 = DiffusionPipeline.from_pretrained(\n",
    "    \"DeepFloyd/IF-II-L-v1.0\", text_encoder=None, variant=\"fp16\", torch_dtype=torch.float16\n",
    ")\n",
    "if useXformers:\n",
    "    stage_2.enable_xformers_memory_efficient_attention()\n",
    "stage_2.enable_model_cpu_offload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "de300d0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "unet\\diffusion_pytorch_model.safetensors not found\n"
     ]
    }
   ],
   "source": [
    "# stage 3\n",
    "#safety_modules = {\"feature_extractor\": stage_1.feature_extractor, \"safety_checker\": stage_1.safety_checker, \"watermarker\": stage_1.watermarker}\n",
    "#stage_3 = DiffusionPipeline.from_pretrained(\"stabilityai/stable-diffusion-x4-upscaler\", **safety_modules, torch_dtype=torch.float16)\n",
    "\n",
    "#del stage_3\n",
    "#flush()\n",
    "stage_3 = DiffusionPipeline.from_pretrained(\"stabilityai/stable-diffusion-x4-upscaler\", torch_dtype=torch.float16)\n",
    "if useXformers:\n",
    "    stage_3.enable_xformers_memory_efficient_attention()\n",
    "stage_3.enable_model_cpu_offload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c23e88a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = 'a photo of a steampunk robot standing in a spaceship holding a sign that says \"IF-XL-v1\"'\n",
    "#prompt = 'a photo of a silly cow standing in a field next to a sign that says \"Moo\"'\n",
    "#prompt = 'A sexy female demon standing in hell holding a sign that says \"Hell\"'\n",
    "#prompt = 'Homer Simpson standing a pub holding a sign that says \"Doh\"'\n",
    "\n",
    "neg_prompt = 'blurry, grainy, unfocused, deformed'\n",
    "\n",
    "# text embeds\n",
    "\n",
    "prompt_embeds, negative_embeds = stage_1.encode_prompt(prompt = prompt, negative_prompt=neg_prompt)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "36a1eb15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b63a535b2bc44da895cf4089eba6f1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "seed=42\n",
    "generator = torch.manual_seed(seed)\n",
    "\n",
    "image = stage_1(prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, generator=generator, output_type=\"pt\").images\n",
    "pt_to_pil(image)[0].save(prompt.replace(\" \",\"_\").replace(\"\\\"\",\"\")+\"-\"+str(seed)+\"if_stage_I.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b2d47b1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4792630929fe4fa1b202189eab8771b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/50 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "image = stage_2(\n",
    "    image=image, prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, generator=generator, output_type=\"pt\"\n",
    ").images\n",
    "pt_to_pil(image)[0].save(prompt.replace(\" \",\"_\").replace(\"\\\"\",\"\")+\"-\"+str(seed)+\"if_stage_II.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7cf9e640",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8931ef9ff042422db09e974e2d6d78e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/75 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "image = stage_3(prompt=prompt, image=image, generator=generator, noise_level=100).images\n",
    "image[0].save(prompt.replace(\" \",\"_\").replace(\"\\\"\",\"\")+\"-\"+str(seed)+\"if_stage_III.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "978e8739",
   "metadata": {},
   "outputs": [
    {
     "data": {