test deploy

.gitignore

.DS_Store
node_modules/
package-lock.json
*.js

App.tsx

import { RefUpdater } from "./RefUpdater";
import { Toolbar } from "./Toolbar";
import { Zoom } from "./Zoom";

export function App() {
  return (
    <div className="w-full relative h-[100dvh]">
      <Zoom />
      <Toolbar />
      <RefUpdater />
    </div>
  );
}


export default App;

atoms.tsx

import { atom } from "jotai";
import { atomWithStorage } from "jotai/utils";
import { BlockType, ModeType, StateRefType } from "./types";
import { defaultRenderPrompt } from "./consts";
import { v4 as uuid } from "uuid";

export const CameraAtom = atom({
  x: 0,
  y: 0,
  z: 1,
});

export const ZoomContainerAtom = atom<HTMLDivElement | null>(null);

export const ModeAtom = atom<ModeType>("move");

export const RenderPromptAtom = atomWithStorage<string>(
  "render-prompt-3",
  defaultRenderPrompt,
);

const starterBlocks = [
  {
    id: uuid(),
    type: "image",
    src: "/cat.jpg",
    x: -400,
    y: -400,
    width: 400,
    height: 300,
  },
  {
    id: uuid(),
    type: "image",
    src: "/pumpkins.jpg",
    x: 200,
    y: -400,
    width: 400,
    height: 300,
  },
  {
    id: uuid(),
    type: "image",
    src: "/clock.jpg",
    x: -200,
    y: 100,
    width: 400,
    height: 300,
  }
];

export const BlockIdsAtom = atom<string[]>(starterBlocks.map((block) => block.id));
let starterBlockMap: Record<string, BlockType> = {};
for (const block of starterBlocks) {
  starterBlockMap[block.id] = block as BlockType;
}
export const BlockMapAtom = atom<Record<string, BlockType>>(starterBlockMap);

export const PromptCreatorAtom = atom<{
  x: number;
  y: number;
  width: number;
  height: number;
} | null>(null);

export const RenderCreatorAtom = atom<{
  x: number;
  y: number;
  width: number;
  height: number;
} | null>(null);

export const SelectedBlockIdsAtom = atom<string[]>([]);

export const BlockSelectorAtom = atom<{
  x: number;
  y: number;
  width: number;
  height: number;
} | null>(null);

export const StateRefAtom = atom<StateRefType>({
  camera: { x: 0, y: 0, z: 1 },
  blockIds: [],
  blockMap: {},
  mode: "move",
  zoomContainer: null,
  selectedBlockIds: [],
  blockSelector: null,
});

BlockResizers.tsx

import { useDrag } from "@use-gesture/react";
import { useAtom } from "jotai";
import { useRef } from "react";
import { BlockMapAtom, CameraAtom, ZoomContainerAtom } from "./atoms";
import { makeZIndex } from "./utils";
import { screenToCanvas } from "./Camera";

export function BlockResizers({ id }: { id: string }) {
  const [blockMap, setBlockMap] = useAtom(BlockMapAtom);
  const block = blockMap[id];

  const [camera] = useAtom(CameraAtom);
  const cameraRef = useRef(camera);
  cameraRef.current = camera;
  const [zoomContainer] = useAtom(ZoomContainerAtom);
  const zoomContainerRef = useRef(zoomContainer);
  zoomContainerRef.current = zoomContainer;

  const preserveAspectRatio = block.type === "image";

  const size = 24;

  const keepCornerRef = useRef({ x: block.x, y: block.y });

  const dirs = ["nw", "ne", "sw", "se"];
  const binds = dirs.map((dir) => {
    return useDrag(({ first, event, xy: [x, y] }) => {
      event.stopPropagation();

      if (first) {
        if (dir === "nw") {
          keepCornerRef.current = {
            x: block.x + block.width,
            y: block.y + block.height,
          };
        } else if (dir === "ne") {
          keepCornerRef.current = {
            x: block.x,
            y: block.y + block.height,
          };
        } else if (dir === "sw") {
          keepCornerRef.current = {
            x: block.x + block.width,
            y: block.y,
          };
        } else if (dir === "se") {
          keepCornerRef.current = {
            x: block.x,
            y: block.y,
          };
        }
      } else {
        const canvasPoint = screenToCanvas(
          { x, y },
          cameraRef.current,
          zoomContainerRef.current!,
        );

        const minX =
          dir === "ne" || dir === "se"
            ? keepCornerRef.current.x
            : canvasPoint.x;
        const minY =
          dir === "sw" || dir === "se"
            ? keepCornerRef.current.y
            : canvasPoint.y;
        const maxX =
          dir === "nw" || dir === "sw"
            ? keepCornerRef.current.x
            : canvasPoint.x;
        const maxY =
          dir === "nw" || dir === "ne"
            ? keepCornerRef.current.y
            : canvasPoint.y;

        let startX = minX;
        let startY = minY;
        let endX = maxX;
        let endY = maxY;

        if (preserveAspectRatio) {
          const aspectRatio = block.width / block.height;
          const newWidth = endX - startX;
          const newHeight = endY - startY;
          const newAspectRatio = newWidth / newHeight;
          if (newAspectRatio > aspectRatio) {
            if (dir === "nw" || dir === "sw") {
              startX = endX - newHeight * aspectRatio;
            } else {
              endX = startX + newHeight * aspectRatio;
            }
          } else {
            if (dir === "nw" || dir === "ne") {
              startY = endY - newWidth / aspectRatio;
            } else {
              endY = startY + newWidth / aspectRatio;
            }
          }
        }

        let width = endX - startX;
        let height = endY - startY;
        const minSize = 48;

        if (width < minSize) {
          if (dir === "nw" || dir === "sw") {
            startX = endX - minSize;
          } else {
            endX = startX + minSize;
          }
          width = endX - startX;
        }
        if (height < minSize) {
          if (dir === "nw" || dir === "ne") {
            startY = endY - minSize;
          } else {
            endY = startY + minSize;
          }
          height = endY - startY;
        }

        setBlockMap((prev) => ({
          ...prev,
          [id]: {
            ...block,
            x: startX,
            y: startY,
            width,
            height,
            zIndex: makeZIndex(),
          },
        }));
      }
    });
  });

  return (
    <>
      <div
        {...binds[0]()}
        className="absolute touch-none pointer-events-auto"
        style={{
          left: -size / 2,
          top: -size / 2,
          cursor: "nwse-resize",
          width: size,
          height: size,
        }}
      />
      <div
        {...binds[1]()}
        className="absolute touch-none pointer-events-auto"
        style={{
          right: -size / 2,
          top: -size / 2,
          cursor: "nesw-resize",
          width: size,
          height: size,
        }}
      />
      <div
        {...binds[2]()}
        className="absolute touch-none pointer-events-auto"
        style={{
          left: -size / 2,
          bottom: -size / 2,
          cursor: "nesw-resize",
          width: size,
          height: size,
        }}
      />
      <div
        {...binds[3]()}
        className="absolute touch-none pointer-events-auto"
        style={{
          right: -size / 2,
          bottom: -size / 2,
          cursor: "nwse-resize",
          width: size,
          height: size,
        }}
      />
    </>
  );
}

Blocks.tsx

import { useAtom } from "jotai";
import {
  BlockIdsAtom,
  BlockMapAtom,
  SelectedBlockIdsAtom,
} from "./atoms";
import { RenderBlockType, ImageBlockType, PromptBlockType } from "./types";
import { PromptBlock } from "./PromptBlock";
import { RenderBlock } from "./RenderBlock";
import { ImageBlock } from "./ImageBlock";
import { BlockResizers } from "./BlockResizers";

export function Blocks() {
  const [blockIds] = useAtom(BlockIdsAtom);
  const [blockMap] = useAtom(BlockMapAtom);
  return (
    <>
      <div className="absolute z-0">
        {blockIds.map((id) => {
          const block = blockMap[id];
          if (block.type !== "render" && block.type !== "prompt") {
            return <BlockWrapper key={id} id={id} />;
          }
        })}
      </div>
      <div className="absolute z-0">
        {blockIds.map((id) => {
          const block = blockMap[id];
          if (block.type === "prompt") {
            return <BlockWrapper key={id} id={id} />;
          }
        })}
      </div>
      <div className="absolute z-0">
        {blockIds.map((id) => {
          const block = blockMap[id];
          if (block.type === "render") {
            return <BlockWrapper key={id} id={id} />;
          }
        })}
      </div>
    </>
  );
}

export function BlockWrapper({ id }: { id: string }) {
  const [blockMap] = useAtom(BlockMapAtom);
  const block = blockMap[id];
  const [selectedBlockIds] = useAtom(SelectedBlockIdsAtom);
  const isSelected = selectedBlockIds.includes(id);

  return (
    <div
      className={`absolute border-2 ${isSelected ? "border-blue-500" : "border-transparent"} touch-none pointer-events-auto`}
      style={{
        left: block.x,
        top: block.y,
        width: block.width,
        height: block.height,
        zIndex: block.zIndex,
        pointerEvents: block.type === "render" ? "none" : "auto",
      }}
    >
      <BlockFactory id={id} />
      <BlockResizers id={id} />
    </div>
  );
}

export function BlockFactory({ id }: { id: string }) {
  const [blockMap] = useAtom(BlockMapAtom);
  const block = blockMap[id];

  switch (block.type) {
    case "image":
      return <ImageBlock block={block as ImageBlockType} />;
    case "render":
      return <RenderBlock block={block as RenderBlockType} />;
    case "prompt":
      return <PromptBlock block={block as PromptBlockType} />;
  }
}

Camera.tsx

import { CameraType, PointType } from "./types";

export function screenToCanvas(
  point: PointType,
  camera: CameraType,
  container: HTMLDivElement,
) {
  const x = (point.x - container.clientWidth / 2) / camera.z - camera.x;
  const y = (point.y - container.clientHeight / 2) / camera.z - camera.y;
  return { x, y };
}

export function canvasToScreen(
  point: PointType,
  camera: CameraType,
  container: HTMLDivElement,
) {
  const x = (point.x + camera.x) * camera.z + container.clientWidth / 2;
  const y = (point.y + camera.y) * camera.z + container.clientHeight / 2;
  return { x, y };
}

export function panCamera(
  camera: CameraType,
  dx: number,
  dy: number,
): CameraType {
  return {
    x: camera.x - dx / camera.z,
    y: camera.y - dy / camera.z,
    z: camera.z,
  };
}

export function zoomCamera(
  camera: CameraType,
  point: PointType,
  dz: number,
  container: HTMLDivElement,
): CameraType {
  const zoom = camera.z - dz * camera.z;

  const p1 = screenToCanvas(point, camera, container);

  const p2 = screenToCanvas(point, { ...camera, z: zoom }, container);

  return {
    x: camera.x + p2.x - p1.x,
    y: camera.y + p2.y - p1.y,
    z: zoom,
  };
}

cat.jpg

clock.jpg

consts.tsx

export const maxSize = 512;
export const defaultRenderPrompt =
  "Generate an image based on the contents provided by the user. If the user provides a collage of styles, combine them into one coherent image in a high-fidelity style. Move and reposition subjects if needed to match the user's intent.";

hooks.tsx

import { useAtom } from "jotai";
import {
  BlockIdsAtom,
  BlockMapAtom,
  CameraAtom,
  ZoomContainerAtom,
} from "./atoms";
import { BlockType } from "./types";
import { useEffect, useRef } from "react";
import { v4 as uuid } from "uuid";
import { screenToCanvas } from "./Camera";
import { loadImage, makeZIndex } from "./utils";
import { maxSize } from "./consts";

export function useCreateBlock() {
  const [, setBlockIds] = useAtom(BlockIdsAtom);
  const [, setBlockMap] = useAtom(BlockMapAtom);

  return (block: BlockType) => {
    setBlockIds((prev) => [...prev, block.id]);
    setBlockMap((prev) => ({ ...prev, [block.id]: block }));
  };
}

export function useUpdateBlock() {
  const [, setBlockMap] = useAtom(BlockMapAtom);

  return (id: string, updates: Partial<BlockType>) => {
    setBlockMap((prev) => ({
      ...prev,
      [id]: { ...prev[id], ...updates } as BlockType,
    }));
  };
}

export function useHandleDropImage() {
  const createBlock = useCreateBlock();
  const [camera] = useAtom(CameraAtom);
  const [zoomContainer] = useAtom(ZoomContainerAtom);

  const cameraRef = useRef(camera);
  cameraRef.current = camera;
  const zoomContainerRef = useRef<HTMLDivElement | null>(null);
  zoomContainerRef.current = zoomContainer;

  function handleDragOver(event: React.DragEvent<HTMLDivElement>) {
    event.preventDefault();
  }

  function handleDropImage(event: React.DragEvent<HTMLDivElement>) {
    event.preventDefault();
    const file = event.dataTransfer?.files[0];
    if (file) {
      const reader = new FileReader();
      reader.onload = async () => {
        const src = reader.result as string;
        const image = await loadImage(src);
        const scale = Math.min(maxSize / image.width, maxSize / image.height);
        const canvasPoint = screenToCanvas(
          { x: event.clientX, y: event.clientY },
          cameraRef.current,
          zoomContainerRef.current!,
        );
        createBlock({
          id: uuid(),
          type: "image",
          src,
          x: canvasPoint.x - (image.width * scale) / 2,
          y: canvasPoint.y - (image.height * scale) / 2,
          width: image.width * scale,
          height: image.height * scale,
          zIndex: makeZIndex(),
        });
      };
      reader.readAsDataURL(file);
    }
  }

  useEffect(() => {
    window.addEventListener("drop", handleDropImage);
    window.addEventListener("dragover", handleDragOver);
    return () => {
      window.removeEventListener("drop", handleDropImage);
      window.removeEventListener("dragover", handleDragOver);
    };
  }, []);
}

export function useHandlePasteImage() {
  const createBlock = useCreateBlock();
  const [camera] = useAtom(CameraAtom);
  const [zoomContainer] = useAtom(ZoomContainerAtom);

  const cameraRef = useRef(camera);
  cameraRef.current = camera;
  const zoomContainerRef = useRef<HTMLDivElement | null>(null);
  zoomContainerRef.current = zoomContainer;

  function handlePasteImage(event: ClipboardEvent) {
    const file = event.clipboardData?.files[0];
    if (file) {
      const reader = new FileReader();
      reader.onload = async () => {
        const src = reader.result as string;
        const image = await loadImage(src);
        const scale = Math.min(maxSize / image.width, maxSize / image.height);
        const canvasPoint = screenToCanvas(
          {
            x: zoomContainerRef.current!.clientWidth / 2,
            y: zoomContainerRef.current!.clientHeight / 2,
          },
          cameraRef.current,
          zoomContainerRef.current!,
        );
        createBlock({
          id: uuid(),
          type: "image",
          src,
          x: canvasPoint.x - (image.width * scale) / 2,
          y: canvasPoint.y - (image.height * scale) / 2,
          width: image.width * scale,
          height: image.height * scale,
          zIndex: makeZIndex(),
        });
      };
      reader.readAsDataURL(file);
    }
  }

  useEffect(() => {
    window.addEventListener("paste", handlePasteImage);
    return () => {
      window.removeEventListener("paste", handlePasteImage);
    };
  }, []);
}

ImageBlock.tsx

import { useEffect, useMemo, useState } from "react";
import { ImageBlockType } from "./types";

export function ImageBlock({ block }: { block: ImageBlockType }) {
  const [imageUrl, setImageUrl] = useState<string | null>(null);

  useEffect(() => {
    async function main() {
      const noStartingSlash = block.src.startsWith("/") ? block.src.slice(1) : block.src;
      const fetched = await fetch(noStartingSlash);
      const blob = await fetched.blob();
      const url = URL.createObjectURL(blob);
      setImageUrl(url);
    }
    main();
  }, [block.src]);

  return (
    <img
      draggable={false}
      src={imageUrl || ''}
      style={{ width: "100%", height: "100%" }}
    />
  );
}

index.css

@tailwind base;
@tailwind components;
@tailwind utilities;

:root {
  color-scheme: dark;
}

html, body, #root {
  height: 100dvh;
  font-size: 16px;
  overflow: hidden;
  font-family: monospace;
}

body {
  background-color: theme('colors.neutral.900');
  color: theme('colors.neutral.200');
}

input, textarea {
  background-color: theme('colors.neutral.800');
  color: theme('colors.neutral.200');
}

.border, .border-l, .border-t, .border-b, .border-r {
  border-color: theme('colors.neutral.700');
}

index.html

<!doctype html>
<html lang="en">
  <head>
    <meta charset="UTF-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <meta http-equiv="X-UA-Compatible" content="ie=edge" />
    <title>App</title>
    <script type="importmap">
{
  "imports": {
    "@mediapipe/tasks-vision": "https://esm.sh/@mediapipe/tasks-vision",
    "@mediapipe/tasks-vision/": "https://esm.sh/@mediapipe/tasks-vision/",
    "@types/uuid": "https://esm.sh/@types/uuid",
    "@types/uuid/": "https://esm.sh/@types/uuid/",
    "@use-gesture/react": "https://esm.sh/@use-gesture/react",
    "@use-gesture/react/": "https://esm.sh/@use-gesture/react/",
    "jotai": "https://esm.sh/jotai",
    "jotai/": "https://esm.sh/jotai/",
    "react": "https://esm.sh/react",
    "react/": "https://esm.sh/react/",
    "react-dom": "https://esm.sh/react-dom",
    "react-dom/": "https://esm.sh/react-dom/",
    "uuid": "https://esm.sh/uuid",
    "uuid/": "https://esm.sh/uuid/",
    "@tailwindcss/browser": "https://esm.sh/@tailwindcss/browser",
    "@tailwindcss/browser/": "https://esm.sh/@tailwindcss/browser/"
  }
}
</script>
    <script type="application/javascript">
      if (window.location.hostname === "localhost") {
        let API_KEY = localStorage.getItem("GEMINI_API_KEY");
        if (!API_KEY) {
          API_KEY = prompt("Please enter your API key from AI Studio:");
          if (API_KEY) {
            localStorage.setItem("GEMINI_API_KEY", API_KEY);
          }
        }
        globalThis.process = { env: { API_KEY: API_KEY } };
      }
    </script>
  </head>

  <body>
    <div id="root"></div>
    <script type="module" src="index.js"></script>
  </body>
</html>

index.tsx

import { StrictMode } from 'react'
import { createRoot } from 'react-dom/client'
import App from "./App"
import '@tailwindcss/browser'

createRoot(document.getElementById('root')!).render(
  <StrictMode>
    <App />
  </StrictMode>,
)

package.json

{
  "name": "helloapplet",
  "version": "1.0.0",
  "main": "index.tsx",
  "scripts": {
    "importmap": "node -e \"const fs = require('fs'); const pkg = JSON.parse(fs.readFileSync('package.json', 'utf8')); let indexHtml = fs.readFileSync('index.html', 'utf8'); const importMap = { imports: {} }; Object.keys(pkg.dependencies || {}).forEach(k => { importMap.imports[k] = 'https://esm.sh/' + k; importMap.imports[k + '/'] = 'https://esm.sh/' + k + '/'; }); const importMapString = JSON.stringify(importMap, null, 2); const regex = /<script\\s+type=\\\"importmap\\\">([\\s\\S]*?)<\\/script>/i; indexHtml = indexHtml.replace(regex, '<script type=\\\"importmap\\\">\\n'+importMapString+'\\n</script>'); fs.writeFileSync('index.html', indexHtml, 'utf8');\"",
    "start": "npm run importmap && tsc --watch",
    "dev": "vite"
  },
  "author": "",
  "license": "Apache-2.0",
  "description": "",
  "devDependencies": {
    "http-server": "^14.1.1",
    "typescript": "^5.8.2",
    "vite": "^6.2.4"
  },
  "dependencies": {
    "@mediapipe/tasks-vision": "^0.10.0",
    "@types/uuid": "^10.0.0",
    "@use-gesture/react": "^10.3.1",
    "jotai": "^2.12.2",
    "react": "^18.3.1",
    "react-dom": "^18.3.1",
    "uuid": "^11.1.0",
    "@tailwindcss/browser": "^4.0.17"
  }
}

PromptBlock.tsx

import { useSetAtom } from "jotai";
import { BlockMapAtom } from "./atoms";
import { PromptBlockType } from "./types";
import { useLayoutEffect, useRef } from "react";

export function PromptBlock({ block }: { block: PromptBlockType }) {
  const setBlockMap = useSetAtom(BlockMapAtom);
  const doubleClickedRef = useRef(false);

  function updateBlock(id: string, updates: Partial<PromptBlockType>) {
    setBlockMap((prev) => ({
      ...prev,
      [id]: { ...prev[id], ...updates } as PromptBlockType,
    }));
  }

  const setBlock = (updates: Partial<PromptBlockType>) => {
    setBlockMap((prev) => ({
      ...prev,
      [block.id]: { ...block, ...updates },
    }));
  };

  const textSizingRef = useRef<HTMLDivElement | null>(null);
  useLayoutEffect(() => {
    if (!textSizingRef.current) return;
    const height = textSizingRef.current.offsetHeight;
    const topBar = 0;
    if (block.height < height + topBar) {
      updateBlock(block.id, { height: height + topBar });
    }
  }, [block.text, block.width, block.height]);

  // TODO placeholder text
  return (
    <div
      className="absolute overflow-hidden"
      style={{ width: "100%", height: "100%" }}
    >
      <div className="absolute inset-0 flex flex-col bg-neutral-800 bg-opacity-80 rounded-xl">
        <div className="uppercase hidden text-xs px-3 pt-2">PROMPT</div>
        <div
          className="absolute pointer-events-none left-0 top-0 w-full px-3 py-2 whitespace-pre-wrap break-words select-none opacity-0 rounded-xl"
          ref={textSizingRef}
        >
          {block.text}&#xfeff;
        </div>
        <div className="relative grow">
          {block.editing ? (
            <textarea
              autoFocus
              className="absolute left-0 top-0 px-3 py-2 w-full h-full focus:bg-transparent focus:outline-none resize-none"
              placeholder="Your prompt"
              value={block.text}
              onFocus={(e) => {
                if (doubleClickedRef.current) {
                  // put cursor at end of text
                  e.target.selectionStart = e.target.selectionEnd =
                    e.target.value.length;
                  doubleClickedRef.current = false;
                }
              }}
              onPointerDown={(e) => e.stopPropagation()}
              onChange={(e) => setBlock({ text: e.target.value })}
              onBlur={() => setBlock({ editing: false })}
            />
          ) : (
            <div
              className="absolute left-0 top-0 w-full px-3 py-2 whitespace-pre-wrap break-words h-full select-none"
              onDoubleClick={() => {
                doubleClickedRef.current = true;
                setBlock({ editing: true });
              }}
            >
              <div className="">{block.text}</div>
            </div>
          )}
        </div>
      </div>
    </div>
  );
}

PromptCreator.tsx

import { useAtom } from "jotai";
import {
  CameraAtom,
  ModeAtom,
  PromptCreatorAtom,
  ZoomContainerAtom,
} from "./atoms";
import { useEffect, useRef } from "react";
import { screenToCanvas } from "./Camera";
import { useCreateBlock } from "./hooks";
import { v4 as uuid } from "uuid";
import { makeZIndex } from "./utils";

export function PromptCreator() {
  const [mode, setMode] = useAtom(ModeAtom);
  const [promptCreator, setPromptCreator] = useAtom(PromptCreatorAtom);
  const createBlock = useCreateBlock();

  const [camera] = useAtom(CameraAtom);
  const [zoomContainer] = useAtom(ZoomContainerAtom);
  const cameraRef = useRef(camera);
  cameraRef.current = camera;
  const zoomContainerRef = useRef<HTMLDivElement | null>(null);
  zoomContainerRef.current = zoomContainer;

  const pointRef = useRef<{ x: number; y: number } | null>(null);

  useEffect(() => {
    if (mode !== "prompt") return;
    function handlePointerDown(e: PointerEvent) {
      (e.target as Element).setPointerCapture(e.pointerId);
      const canvasPoint = screenToCanvas(
        { x: e.clientX, y: e.clientY },
        cameraRef.current,
        zoomContainerRef.current!,
      );
      pointRef.current = { x: canvasPoint.x, y: canvasPoint.y };
    }
    function handlePointerMove(e: PointerEvent) {
      if (e.buttons !== 1) return;
      if (!pointRef.current) return;
      const canvasPoint = screenToCanvas(
        { x: e.clientX, y: e.clientY },
        cameraRef.current,
        zoomContainerRef.current!,
      );
      const x = Math.min(canvasPoint.x, pointRef.current.x);
      const y = Math.min(canvasPoint.y, pointRef.current.y);
      const width = Math.abs(canvasPoint.x - pointRef.current.x);
      const height = Math.abs(canvasPoint.y - pointRef.current.y);
      setPromptCreator({ x, y, width, height });
    }
    function handlePointerUp(e: PointerEvent) {
      (e.target as Element).releasePointerCapture(e.pointerId);

      if (!pointRef.current) return;
      const canvasPoint = screenToCanvas(
        { x: e.clientX, y: e.clientY },
        cameraRef.current,
        zoomContainerRef.current!,
      );
      const x = Math.min(canvasPoint.x, pointRef.current.x);
      const y = Math.min(canvasPoint.y, pointRef.current.y);
      const width = Math.max(Math.abs(canvasPoint.x - pointRef.current.x), 180)
      const height = Math.max(Math.abs(canvasPoint.y - pointRef.current.y), 24)
      pointRef.current = null;

      createBlock({
        id: uuid(),
        type: "prompt",
        x,
        y,
        width,
        height,
        text: "",
        editing: true,
        zIndex: makeZIndex(),
      });
      setPromptCreator(null);
      setMode("move");
    }
    window.addEventListener("pointerdown", handlePointerDown);
    window.addEventListener("pointermove", handlePointerMove);
    window.addEventListener("pointerup", handlePointerUp);
    return () => {
      window.removeEventListener("pointerdown", handlePointerDown);
      window.removeEventListener("pointermove", handlePointerMove);
      window.removeEventListener("pointerup", handlePointerUp);
    };
  }, [mode, zoomContainer]);

  return promptCreator ? (
    <div
      className="border-2 border-blue-500"
      style={{
        position: "absolute",
        left: promptCreator.x,
        top: promptCreator.y,
        width: promptCreator.width,
        height: promptCreator.height,
      }}
    />
  ) : null;
}

pumpkins.jpg

RefUpdater.tsx

import { useAtom } from "jotai";
import { BlockIdsAtom, BlockMapAtom, BlockSelectorAtom, CameraAtom, ModeAtom, SelectedBlockIdsAtom, StateRefAtom, ZoomContainerAtom } from "./atoms";
import { useEffect } from "react";

export function RefUpdater() {
  const [stateRef] = useAtom(StateRefAtom);
  const [camera] = useAtom(CameraAtom);
  const [blockIds] = useAtom(BlockIdsAtom);
  const [blockMap] = useAtom(BlockMapAtom);
  const [mode] = useAtom(ModeAtom);
  const [zoomContainer] = useAtom(ZoomContainerAtom);
  const [selectedBlockIds] = useAtom(SelectedBlockIdsAtom);
  const [blockSelector] = useAtom(BlockSelectorAtom);

  // If you add a new entry make sure you add it to the dependency array
  useEffect(() => {
    stateRef.camera = camera;
    stateRef.blockIds = blockIds;
    stateRef.blockMap = blockMap;
    stateRef.mode = mode;
    stateRef.zoomContainer = zoomContainer;
    stateRef.selectedBlockIds = selectedBlockIds;
  }, [camera, blockIds, blockMap, mode, zoomContainer, selectedBlockIds, blockSelector]);

  return null
}

RenderBlock.tsx

import { useAtom } from "jotai";
import { v4 as uuid } from "uuid";
import {
  BlockIdsAtom,
  BlockMapAtom,
  RenderPromptAtom,
} from "./atoms";
import { maxSize } from "./consts";
import { useCreateBlock, useUpdateBlock } from "./hooks";
import { predict } from "./modelUtils";
import { RenderBlockType } from "./types";
import { loadImage, makeZIndex } from "./utils";
// import { GenerateContentRequest } from "@google/generative-ai";

const sides = ["top", "right", "bottom", "left"] as const;

export function RenderBlock({ block }: { block: RenderBlockType }) {
  const [blockIds] = useAtom(BlockIdsAtom);
  const [blockMap] = useAtom(BlockMapAtom);
  const createBlock = useCreateBlock();
  const updateBlock = useUpdateBlock();
  const [systemInstruction] = useAtom(RenderPromptAtom);

  async function handleRender() {
    const canvas = document.createElement("canvas");
    canvas.width = block.width;
    canvas.height = block.height;
    const ctx = canvas.getContext("2d")!;

    ctx.fillStyle = "black";
    ctx.fillRect(0, 0, block.width, block.height);

    const imageBlocks = blockIds
      .map((id) => blockMap[id])
      .filter((block) => block.type === "image");
    // sort by zIndex
    imageBlocks.sort((a, b) => a.zIndex - b.zIndex);

    let imageActive = false;
    let minX = Infinity;
    let minY = Infinity;
    let maxX = -Infinity;
    let maxY = -Infinity;

    for (const imageBlock of imageBlocks) {
      if (
        imageBlock.x + imageBlock.width < block.x ||
        imageBlock.x > block.x + block.width
      )
        continue;
      if (
        imageBlock.y + imageBlock.height < block.y ||
        imageBlock.y > block.y + block.height
      )
        continue;
      imageActive = true;
      minX = Math.min(minX, imageBlock.x - block.x);
      minY = Math.min(minY, imageBlock.y - block.y);
      maxX = Math.max(maxX, imageBlock.x - block.x + imageBlock.width);
      maxY = Math.max(maxY, imageBlock.y - block.y + imageBlock.height);
      const image = await loadImage(imageBlock.src);
      ctx.drawImage(
        image,
        imageBlock.x - block.x,
        imageBlock.y - block.y,
        imageBlock.width,
        imageBlock.height,
      );
    }

    let image = canvas.toDataURL();
    let croppedImage = null;
    let blurredImage = null;
    if (imageActive) {
      croppedImage = document.createElement("canvas");
      croppedImage.width = maxX - minX;
      croppedImage.height = maxY - minY;
      const croppedCtx = croppedImage.getContext("2d")!;
      croppedCtx.drawImage(
        canvas,
        minX,
        minY,
        maxX - minX,
        maxY - minY,
        0,
        0,
        maxX - minX,
        maxY - minY,
      );
      image = croppedImage.toDataURL();

      const blurredCanvas = document.createElement("canvas");
      blurredCanvas.width = croppedImage.width;
      blurredCanvas.height = croppedImage.height;
      const blurredCtx = blurredCanvas.getContext("2d")!;
      blurredCtx.filter = "blur(16px)";
      blurredCtx.drawImage(croppedImage, 0, 0);
      blurredImage = blurredCanvas.toDataURL();
    }

    // update to use for labels
    // function textWrap(text: string, maxWidth: number) {
    //   const words = text.split(" ");
    //   const lines: string[] = [];
    //   let line = "";
    //   for (const word of words) {
    //     const testLine = line + word + " ";
    //     const testWidth = ctx.measureText(testLine).width;
    //     if (testWidth > maxWidth) {
    //       lines.push(line);
    //       line = word + " ";
    //     } else {
    //       line = testLine;
    //     }
    //   }
    //   lines.push(line);
    //   return lines;
    // }
    // const promptBlocks = blockIds
    //   .map((id) => blockMap[id])
    //   .filter((block) => block.type === "prompt");
    // for (const promptBlock of promptBlocks) {
    //   if (
    //     promptBlock.x + promptBlock.width < block.x ||
    //     promptBlock.x > block.x + block.width
    //   )
    //     continue;
    //   if (
    //     promptBlock.y + promptBlock.height < block.y ||
    //     promptBlock.y > block.y + block.height
    //   )
    //     continue;
    //   ctx.font = "16px sans-serif";
    //   ctx.fillStyle = "white";
    //   const lines = textWrap(promptBlock.text, promptBlock.width);
    //   for (let i = 0; i < lines.length; i++) {
    //     ctx.fillText(
    //       lines[i],
    //       promptBlock.x - block.x,
    //       promptBlock.y - block.y + 16 * (i + 1),
    //     );
    //   }
    // }

    let activePrompts = [];
    const promptBlocks = blockIds
      .map((id) => blockMap[id])
      .filter((block) => block.type === "prompt");
    for (const promptBlock of promptBlocks) {
      if (
        promptBlock.x + promptBlock.width < block.x ||
        promptBlock.x > block.x + block.width
      )
        continue;
      if (
        promptBlock.y + promptBlock.height < block.y ||
        promptBlock.y > block.y + block.height
      )
        continue;
      activePrompts.push(promptBlock.text);
    }

    const id = uuid();

    // visualize debug
    createBlock({
      id,
      type: "image",
      src: blurredImage || image,
      x: block.x + block.width + 16,
      y: block.y,
      width: croppedImage ? croppedImage.width : block.width,
      height: croppedImage ? croppedImage.height : block.height,
      zIndex: makeZIndex(),
    });

    // let contents: GenerateContentRequest = {
    //   contents: [
    //     {
    //       parts: [],
    //       role: "user",
    //     },
    //   ],
    // };
    //
    // if (imageActive) {
    //   contents.contents[0].parts.push({
    //     inlineData: {
    //       mimeType: "image/jpeg",
    //       data: croppedImage!.toDataURL().split(",")[1],
    //     },
    //   });
    // }
    //
    // contents.contents[0].parts.push({
    //   text: "Instructions: " + systemInstruction,
    // });
    //
    // if (activePrompts.length > 0) {
    //   for (const prompt of activePrompts) {
    //     contents.contents[0].parts.push({
    //       text: prompt,
    //     });
    //   }
    // }
    //
    // const responseParts = await predict({
    //   contents,
    // });
    //
    // let lastImagePart = null;
    //
    // for (const part of responseParts) {
    //   if (part.inlineData) {
    //     lastImagePart = part;
    //   }
    // }
    //
    // if (lastImagePart) {
    //   const generatedImageSrc =
    //     "data:" +
    //     lastImagePart.inlineData.mimeType +
    //     ";base64," +
    //     lastImagePart.inlineData.data;
    //   const generatedImage = await loadImage(generatedImageSrc);
    //   const newCanvas = document.createElement("canvas");
    //   const scale = Math.min(
    //     maxSize / generatedImage.width,
    //     maxSize / generatedImage.height,
    //   );
    //   newCanvas.width = generatedImage.width * scale;
    //   newCanvas.height = generatedImage.height * scale;
    //   const newCtx = newCanvas.getContext("2d")!;
    //   newCtx.drawImage(
    //     generatedImage,
    //     0,
    //     0,
    //     generatedImage.width,
    //     generatedImage.height,
    //     0,
    //     0,
    //     newCanvas.width,
    //     newCanvas.height,
    //   );
    //   updateBlock(id, {
    //     type: "image",
    //     src: newCanvas.toDataURL(),
    //     width: newCanvas.width,
    //     height: newCanvas.height,
    //     zIndex: makeZIndex(),
    //   });
    // }
    //
    // setLoadingMessage(null);
  }

  return (
    <div
      className="border-2 border-red-500 absolute"
      style={{
        width: "100%",
        height: "100%",
      }}
    >
      <button
        className="absolute pointer-events-auto -bottom-9 right-0 px-3 py-1 text-sm rounded-lg bg-red-500 hover:bg-red-600 text-white"
        onClick={handleRender}
      >
        Render
      </button>
      {sides.map((side) => (
        <div
          key={side}
          className="absolute cursor-move pointer-events-auto"
          style={{
            [side]: -8,
            width: side === "top" || side === "bottom" ? "100%" : 16,
            height: side === "top" || side === "bottom" ? 16 : "100%",
          }}
        />
      ))}
    </div>
  );
}

RenderCreator.tsx

import { useAtom } from "jotai";
import {
  BlockIdsAtom,
  BlockMapAtom,
  CameraAtom,
  RenderCreatorAtom,
  ModeAtom,
  ZoomContainerAtom,
} from "./atoms";
import { useEffect, useRef } from "react";
import { screenToCanvas } from "./Camera";
import { useCreateBlock } from "./hooks";
import { v4 as uuid } from "uuid";
import { makeZIndex } from "./utils";

export function RenderCreator() {
  const [mode, setMode] = useAtom(ModeAtom);
  const [renderCreator, setRenderCreator] = useAtom(RenderCreatorAtom);
  const createBlock = useCreateBlock();

  const [camera] = useAtom(CameraAtom);
  const [zoomContainer] = useAtom(ZoomContainerAtom);
  const cameraRef = useRef(camera);
  cameraRef.current = camera;
  const zoomContainerRef = useRef<HTMLDivElement | null>(null);
  zoomContainerRef.current = zoomContainer;

  const pointRef = useRef<{ x: number; y: number } | null>(null);

  const [blockIds] = useAtom(BlockIdsAtom);
  const [blockMap] = useAtom(BlockMapAtom);

  const blocks = blockIds.map((id) => blockMap[id]);
  const blocksRef = useRef(blocks);
  blocksRef.current = blocks;

  useEffect(() => {
    if (mode !== "render") return;
    function handlePointerDown(e: PointerEvent) {
      (e.target as Element).setPointerCapture(e.pointerId);
      const point = screenToCanvas(
        { x: e.clientX, y: e.clientY },
        cameraRef.current,
        zoomContainerRef.current!,
      );
      const overBlocks = blocksRef.current
        .filter((block) => block.type === "render")
        .filter((block) => {
          return (
            point.x > block.x &&
            point.x < block.x + block.width &&
            point.y > block.y &&
            point.y < block.y + block.height
          );
        })
        .sort((a, b) => b.zIndex - a.zIndex);

      if (overBlocks.length === 0) {
        pointRef.current = { x: point.x, y: point.y };
      }
    }
    function handlePointerMove(e: PointerEvent) {
      if (e.buttons !== 1) return;
      if (!pointRef.current) return;
      const canvasPoint = screenToCanvas(
        { x: e.clientX, y: e.clientY },
        cameraRef.current,
        zoomContainerRef.current!,
      );
      const x = Math.min(canvasPoint.x, pointRef.current.x);
      const y = Math.min(canvasPoint.y, pointRef.current.y);
      const width = Math.abs(canvasPoint.x - pointRef.current.x);
      const height = Math.abs(canvasPoint.y - pointRef.current.y);
      setRenderCreator({ x, y, width, height });
    }
    function handlePointerUp(e: PointerEvent) {
      (e.target as Element).releasePointerCapture(e.pointerId);

      if (!pointRef.current) return;
      const canvasPoint = screenToCanvas(
        { x: e.clientX, y: e.clientY },
        cameraRef.current,
        zoomContainerRef.current!,
      );
      const x = Math.min(canvasPoint.x, pointRef.current.x);
      const y = Math.min(canvasPoint.y, pointRef.current.y);
      const width = Math.max(Math.abs(canvasPoint.x - pointRef.current.x), 96);
      const height = Math.max(Math.abs(canvasPoint.y - pointRef.current.y), 48);
      pointRef.current = null;

      createBlock({
        id: uuid(),
        type: "render",
        x,
        y,
        width,
        height,
        prompt: "make image",
        zIndex: makeZIndex(),
      });
      setRenderCreator(null);
      setMode("move");
    }
    window.addEventListener("pointerdown", handlePointerDown);
    window.addEventListener("pointermove", handlePointerMove);
    window.addEventListener("pointerup", handlePointerUp);
    return () => {
      window.removeEventListener("pointerdown", handlePointerDown);
      window.removeEventListener("pointermove", handlePointerMove);
      window.removeEventListener("pointerup", handlePointerUp);
    };
  }, [mode, zoomContainer]);

  return renderCreator ? (
    <div
      className="border-2 border-blue-500"
      style={{
        position: "absolute",
        left: renderCreator.x,
        top: renderCreator.y,
        width: renderCreator.width,
        height: renderCreator.height,
      }}
    />
  ) : null;
}

Toolbar.tsx

import { useAtom } from "jotai";
import { useRef, useEffect } from "react";
import { ModeAtom } from "./atoms";
import { ModeType } from "./types";

export function Toolbar() {
  const [mode, setMode] = useAtom(ModeAtom);

  const lastModeRef = useRef<ModeType>(mode);
  useEffect(() => {
    function handleModifierDown(event: KeyboardEvent) {
      if (event.key === "Shift") {
        lastModeRef.current = mode;
        setMode("segment");
      }
    }
    function handleModifierUp(event: KeyboardEvent) {
      if (event.key === "Shift") {
        setMode(lastModeRef.current);
      }
    }
    window.addEventListener("keydown", handleModifierDown);
    window.addEventListener("keyup", handleModifierUp);
    return () => {
      window.removeEventListener("keydown", handleModifierDown);
      window.removeEventListener("keyup", handleModifierUp);
    };
  }, [mode, setMode]);

  return (
    <>
      <div className="absolute left-0 top-0 pointer-events-none">
        <div className="px-3 py-3">Blocks Lite</div>
      </div>
      <div className="flex gap-2 absolute bottom-4 left-1/2 -translate-x-1/2">
        <button
          className={`${mode === "move" ? "bg-neutral-800" : "bg-neutral-700"} px-3 py-1 rounded-lg`}
          onPointerDown={(e) => {
            e.stopPropagation();
          }}
          onClick={() => {
            setMode("move");
          }}
        >
          Move
        </button>
        <button
          className={`${mode === "prompt" ? "bg-neutral-800" : "bg-neutral-700"} px-3 py-1 rounded-lg`}
          onPointerDown={(e) => {
            e.stopPropagation();
          }}
          onClick={() => {
            setMode("prompt");
          }}
        >
          Prompt
        </button>
        <button
          className={`${mode === "segment" ? "bg-neutral-800" : "bg-neutral-700"} px-3 py-1 rounded-lg`}
          onPointerDown={(e) => {
            e.stopPropagation();
          }}
          onClick={() => {
            setMode("segment");
          }}
        >
          Segment
        </button>
      </div>
    </>
  );
}

tsconfig.json

{
    "compilerOptions": {
        "jsx": "react-jsx",
        "skipLibCheck": true,
        "module": "esnext",
        "target": "es2024",
        "moduleResolution": "bundler",
        "typeRoots": [
            "./node_modules/@types"
        ],
        "lib": [
            "DOM",
            "ESNext"
        ]
    }
}

useDragAndSelect.tsx

import { useDrag } from "@use-gesture/react";
import { useAtom } from "jotai";
import {
  BlockIdsAtom,
  BlockMapAtom,
  BlockSelectorAtom,
  ModeAtom,
  SelectedBlockIdsAtom,
  StateRefAtom,
} from "./atoms";
import { useRef } from "react";
import { screenToCanvas } from "./Camera";
import {
  blockIntersectBlocks,
  loadImage,
  makeZIndex,
  pointIntersectBlocks,
} from "./utils";
import { BlockType, ImageBlockType, PointType } from "./types";
import { v4 as uuid } from "uuid";
import { FilesetResolver, InteractiveSegmenter } from "@mediapipe/tasks-vision";

let interactiveSegmenter: InteractiveSegmenter;
const createSegmenter = async () => {
  // Move to CDN
  const filesetResolver = await FilesetResolver.forVisionTasks(
    "https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm",
  );
  interactiveSegmenter = await InteractiveSegmenter.createFromOptions(
    filesetResolver,
    {
      baseOptions: {
        modelAssetPath: `https://storage.googleapis.com/mediapipe-models/interactive_segmenter/magic_touch/float32/1/magic_touch.tflite`,
        delegate: "GPU",
      },
      outputCategoryMask: true,
      outputConfidenceMasks: false,
    },
  );
};
createSegmenter();

export function useDragAndSelect() {
  const [, setBlockSelector] = useAtom(BlockSelectorAtom);
  const [stateRef] = useAtom(StateRefAtom);
  const [, setSelectedBlockIds] = useAtom(SelectedBlockIdsAtom);
  const [, setBlockMap] = useAtom(BlockMapAtom);
  const [, setBlockIds] = useAtom(BlockIdsAtom);
  const [, setMode] = useAtom(ModeAtom);

  const initialPositionsRef = useRef<PointType[]>([]);
  const initialPointRef = useRef({ x: 0, y: 0 });

  return useDrag(async ({ first, last, xy: [x, y] }) => {
    const canvasPoint = screenToCanvas(
      { x, y },
      stateRef.camera,
      stateRef.zoomContainer!,
    );
    const currentBlocks = stateRef.blockIds.map((id) => stateRef.blockMap[id]);
    if (first) {
      initialPointRef.current = { x: canvasPoint.x, y: canvasPoint.y };
    }

    switch (stateRef.mode) {
      case "move": {
        if (first) {
          const intersected = pointIntersectBlocks(canvasPoint, currentBlocks);

          if (intersected.length > 0) {
            let promptBlocks = intersected.filter(
              (block) => block.type === "prompt",
            );
            promptBlocks = promptBlocks.sort((a, b) => b.zIndex - a.zIndex);
            let imageBlocks = intersected.filter(
              (block) => block.type === "image",
            );
            imageBlocks = imageBlocks.sort((a, b) => b.zIndex - a.zIndex);
            const sortedBlocks = [...promptBlocks, ...imageBlocks];

            const topBlock = sortedBlocks[0].id;

            if (stateRef.selectedBlockIds.includes(topBlock)) {
              // keep selection
            } else {
              stateRef.selectedBlockIds = [topBlock];
              setSelectedBlockIds([topBlock]);
            }
            initialPositionsRef.current = stateRef.selectedBlockIds.map(
              (id) => {
                const block = stateRef.blockMap[id];
                return {
                  x: block.x,
                  y: block.y,
                };
              },
            );
          } else {
            stateRef.selectedBlockIds = [];
            setSelectedBlockIds([]);
          }
        }

        if (stateRef.selectedBlockIds.length === 0) {
          const minX = Math.min(canvasPoint.x, initialPointRef.current.x);
          const minY = Math.min(canvasPoint.y, initialPointRef.current.y);
          const maxX = Math.max(canvasPoint.x, initialPointRef.current.x);
          const maxY = Math.max(canvasPoint.y, initialPointRef.current.y);
          const blockSelector = {
            x: minX,
            y: minY,
            width: maxX - minX,
            height: maxY - minY,
          };
          stateRef.blockSelector = blockSelector;
          setBlockSelector(blockSelector);
        } else {
          // moveBlocks
          const selectedBlocks = stateRef.selectedBlockIds.map(
            (id) => stateRef.blockMap[id],
          );
          let newBlockObj: Record<string, BlockType> = {};
          for (let i = 0; i < selectedBlocks.length; i++) {
            const block = selectedBlocks[i];
            newBlockObj[block.id] = {
              ...block,
              x:
                initialPositionsRef.current[i].x +
                (canvasPoint.x - initialPointRef.current.x),
              y:
                initialPositionsRef.current[i].y +
                (canvasPoint.y - initialPointRef.current.y),
              zIndex: makeZIndex() + 1,
            } as BlockType;
          }
          setBlockMap((prev) => {
            return { ...prev, ...newBlockObj };
          });
        }

        if (last) {
          if (stateRef.blockSelector) {
            const _selectedBlocks = blockIntersectBlocks(
              stateRef.blockSelector as BlockType,
              currentBlocks,
            );
            let promptBlocks = _selectedBlocks.filter(
              (block) => block.type === "prompt",
            );
            promptBlocks = promptBlocks.sort((a, b) => b.zIndex - a.zIndex);
            let imageBlocks = _selectedBlocks.filter(
              (block) => block.type === "image",
            );
            imageBlocks = imageBlocks.sort((a, b) => b.zIndex - a.zIndex);
            const sortedBlocks = [...promptBlocks, ...imageBlocks];
            const sortedBlocksIds = sortedBlocks.map((block) => block.id);

            stateRef.selectedBlockIds = sortedBlocksIds;
            setSelectedBlockIds(sortedBlocksIds);
            stateRef.blockSelector = null;
            setBlockSelector(null);
          }
        }
        break;
      }
      case "segment": {
        if (first) {
          const intersected = pointIntersectBlocks(canvasPoint, currentBlocks);
          let intersectedImages = intersected.filter(
            (block) => block.type === "image",
          );

          if (intersectedImages.length > 0) {
            intersectedImages = intersectedImages.sort(
              (a, b) => b.zIndex - a.zIndex,
            );
            const topBlock = intersectedImages[0];

            const imageCanvas = document.createElement("canvas");
            imageCanvas.width = topBlock.width;
            imageCanvas.height = topBlock.height;

            const image = await loadImage(topBlock.src);
            const ctx = imageCanvas.getContext("2d")!;
            ctx.drawImage(image, 0, 0, imageCanvas.width, imageCanvas.height);

            const imageData = ctx.getImageData(
              0,
              0,
              imageCanvas.width,
              imageCanvas.height,
            );

            interactiveSegmenter.segment(
              imageCanvas,
              {
                keypoint: {
                  x: (canvasPoint.x - topBlock.x) / topBlock.width,
                  y: (canvasPoint.y - topBlock.y) / topBlock.height,
                },
              },
              (result) => {
                const mask = result.categoryMask;
                if (!mask) return;
                const resultCanvas = document.createElement("canvas");
                resultCanvas.width = topBlock.width;
                resultCanvas.height = topBlock.height;
                const resultCtx = resultCanvas.getContext("2d")!;
                resultCtx.drawImage(
                  imageCanvas,
                  0,
                  0,
                  topBlock.width,
                  topBlock.height,
                );
                const tempData = resultCtx.getImageData(
                  0,
                  0,
                  resultCanvas.width,
                  resultCanvas.height,
                );
                let minX = resultCanvas.width;
                let minY = resultCanvas.height;
                let maxX = 0;
                let maxY = 0;
                const maskData = mask.getAsFloat32Array();
                for (let i = 0; i < tempData.data.length; i += 4) {
                  if (Math.round(maskData[i / 4] * 255.0) === 0) {
                    const x = (i / 4) % resultCanvas.width;
                    const y = Math.floor(i / 4 / resultCanvas.width);
                    minX = Math.min(minX, x);
                    minY = Math.min(minY, y);
                    maxX = Math.max(maxX, x);
                    maxY = Math.max(maxY, y);
                    tempData.data[i] = imageData.data[i];
                    tempData.data[i + 1] = imageData.data[i + 1];
                    tempData.data[i + 2] = imageData.data[i + 2];
                  } else {
                    tempData.data[i + 3] = 0;
                  }
                }
                resultCtx.putImageData(tempData, 0, 0);

                const finalCanvas = document.createElement("canvas");
                finalCanvas.width = maxX - minX;
                finalCanvas.height = maxY - minY;
                const finalCtx = finalCanvas.getContext("2d")!;
                finalCtx.drawImage(
                  resultCanvas,
                  minX,
                  minY,
                  finalCanvas.width,
                  finalCanvas.height,
                  0,
                  0,
                  finalCanvas.width,
                  finalCanvas.height,
                );
                const finalSrc = finalCanvas.toDataURL();

                const newId = uuid();

                const newBlock = {
                  id: newId,
                  type: "image",
                  src: finalSrc,
                  x: topBlock.x + minX,
                  y: topBlock.y + minY,
                  width: maxX - minX,
                  height: maxY - minY,
                  zIndex: makeZIndex(),
                } as ImageBlockType;

                const newMap = { ...stateRef.blockMap, [newId]: newBlock };
                const newIds = [...stateRef.blockIds, newId];
                stateRef.blockMap = newMap;
                setBlockMap(newMap);
                stateRef.blockIds = newIds;
                setBlockIds(newIds);
                stateRef.selectedBlockIds = [newId];
                setSelectedBlockIds([newId]);
                initialPositionsRef.current = [
                  {
                    x: topBlock.x + minX,
                    y: topBlock.y + minY,
                  },
                ];
                stateRef.mode = "move";
                setMode("move");
              },
            );
          }
        }
        break;
      }
      default: {
        break;
      }
    }
  });
}

utils.tsx

import { BlockType, PointType } from "./types";

// load image as promise
export function loadImage(src: string): Promise<HTMLImageElement> {
  return new Promise((resolve, reject) => {
    const img = new Image();
    img.onload = () => resolve(img);
    img.onerror = reject;
    img.src = src;
  });
}

export function makeZIndex() {
  return Math.round((Date.now() - 1729536285367) / 100);
}

export function pointIntersectBlocks({ x, y }: PointType, blocks: BlockType[]) {
  return blocks.filter((block) => {
    return (
      x >= block.x &&
      x <= block.x + block.width &&
      y >= block.y &&
      y <= block.y + block.height
    );
  });
}

// TODO make this work
export function blockIntersectBlocks(
  { x, y, width, height }: BlockType,
  blocks: BlockType[],
) {
  return blocks.filter((block) => {
    return (
      x < block.x + block.width &&
      x + width > block.x &&
      y < block.y + block.height &&
      y + height > block.y
    );
  });
}

vite-env.d.ts

/// <reference types="vite/client" />

vite.svg

Zoom.tsx

import { useAtom } from "jotai";
import { useEffect } from "react";
import { BlockSelectorAtom, CameraAtom, ZoomContainerAtom } from "./atoms";
import { Blocks } from "./Blocks";
import { useHandleDropImage, useHandlePasteImage } from "./hooks";
import { RenderCreator } from "./RenderCreator";
import { PromptCreator } from "./PromptCreator";
import { panCamera, zoomCamera } from "./Camera";
import { useDragAndSelect } from "./useDragAndSelect";

export function Zoom() {
  const [camera, setCamera] = useAtom(CameraAtom);
  const [zoomContainer, setZoomContainer] = useAtom(ZoomContainerAtom);
  useHandleDropImage();
  useHandlePasteImage();
  const useDragBind = useDragAndSelect();

  useEffect(() => {
    function handleWheel(event: WheelEvent) {
      if (zoomContainer) {
        event.preventDefault();

        const { clientX: x, clientY: y, deltaX, deltaY, ctrlKey } = event;

        if (ctrlKey) {
          setCamera((camera) =>
            zoomCamera(camera, { x, y }, deltaY / 400, zoomContainer),
          );
        } else {
          if (event.shiftKey) {
            setCamera((camera) => panCamera(camera, deltaY, 0));
          } else {
            setCamera((camera) => panCamera(camera, deltaX, deltaY));
          }
        }
      }
    }
    window.addEventListener("wheel", handleWheel, { passive: false });
    return () => window.removeEventListener("wheel", handleWheel);
  }, [zoomContainer, setCamera]);

  return (
    <div
      {...useDragBind()}
      className="absolute inset-0 touch-none"
      ref={(div) => {
        if (div) {
          setZoomContainer(div);
        }
      }}
    >
      <div
        style={{
          position: "absolute",
          left: "50%",
          top: "50%",
          width: "100%",
          height: "100%",
          transformOrigin: "0 0",
          transform: `scale(${camera.z}) translate(-50%, -50%) translate(${camera.x}px, ${camera.y}px)`,
          display: "flex",
          justifyContent: "center",
          alignItems: "center",
        }}
      >
        <div className="relative pointer-events-none">
          <Blocks />
          <RenderCreator />
          <PromptCreator />
          <BlockSelector />
        </div>
      </div>
    </div>
  );
}

function BlockSelector() {
  const [blockSelector] = useAtom(BlockSelectorAtom);

  return (
    blockSelector ? <div
      className="absolute pointer-events-none border-[2px] border-blue-500"
      style={{
        left: blockSelector.x,
        top: blockSelector.y,
        width: blockSelector.width,
        height: blockSelector.height,
      }}
    /> : null
  );
}