Best practice: Full UGC video pipeline with varg SDK — talking character, lipsync, food slideshow, captions

rabbi-kosher-full.tsx

/**
 * @jsxImportSource vargai
 *
 * BEST PRACTICE: Full UGC video pipeline with varg SDK
 *
 * This script demonstrates the recommended pattern for creating talking-character
 * UGC videos with the varg React engine. Key patterns shown:
 *
 * 1. SPEECH-FIRST WORKFLOW
 *    - Use a single `await Speech({ children: [...] })` call with all lines
 *    - Destructure `{ audio, segments }` to get the full audio + per-line segments
 *    - Each segment has `.duration` (probed via ffprobe) for precise clip timing
 *
 * 2. CHARACTER CONSISTENCY
 *    - Define a character reference image (RABBI_REF) and a text description (CHAR)
 *    - Use `nano-banana-pro/edit` with `images: [reference]` to keep the character
 *      identical across multiple angles (wide, close-up, kitchen)
 *    - Chain images: rabbiCloseup uses rabbiWide as additional reference
 *
 * 3. LIPSYNC TALKING HEAD
 *    - Use `Video({ model: varg.videoModel("veed-fabric-1.0"), prompt: { images, audio } })`
 *    - Pass the speech segment as `audio` — the model generates a talking video
 *    - Set `keepAudio: true` to preserve the speech audio in the output video
 *
 * 4. MIXED CLIP TYPES
 *    - Lipsync clips: `<Clip duration={segments[i].duration}>{talkVideo}</Clip>`
 *    - Image + voiceover clips: `<Clip duration={segments[i].duration}>{image}{segments[i]}</Clip>`
 *    - Duration always comes from `segments[i].duration` for tight audio sync
 *
 * 5. MUSIC + CAPTIONS
 *    - Background music with `ducking` auto-lowers under speech
 *    - `<Captions src={audio}>` generates TikTok-style captions from the full audio
 *
 * 6. UPLOAD (optional)
 *    - After render, upload the final video to varg storage using VargClient.uploadFile()
 *    - See the upload example at the bottom of this file
 *
 * Required env: VARG_API_KEY
 * Run: mkdir -p output && bun scripts/rabbi-kosher-full.tsx
 */
import { render, Render, Clip, Music, Image, Video, Speech, Captions } from "vargai/react"
import { createVarg, VargClient } from "@vargai/gateway"
import { mkdirSync } from "fs"

mkdirSync("output", { recursive: true })

const varg = createVarg({ apiKey: process.env.VARG_API_KEY! })

const RABBI_REF = "https://s3.varg.ai/u/cb40f707-37a1-4b52-848f-1334d66b3ad8/274747a19246936edf444cd2218adf738bf1850f737f043227f0e7c2f243c945.bin";
const CHAR = `Use the provided character reference. Maximum consistency — same beard, same wise aged face, same black hat, same dark clothing. Identical rabbi in every shot.`;
const VOICE = "BOhNMSHwH7qAbX1A4Xyj";
const speechModel = varg.speechModel("eleven_v3");

// ── Speech — single call, all lines ──
const { audio, segments } = await Speech({
  model: speechModel,
  voice: VOICE,
  children: [
    "Shalom!",                                                                          // 0 — wide
    "What is kosher?",                                                                   // 1 — close-up
    "Kosher means fit. It is how Jews eat according to the Torah.",                       // 2 — food montage
    "Beef.",                                                                            // 3 — beef
    "Chicken.",                                                                          // 4 — chicken
    "Salmon.",                                                                           // 5 — salmon
    "Fruits and vegetables.",                                                            // 6 — fruits and vegetables
    "Eggs and challah bread.",                                                            // 7 — eggs and challah bread
    "All kosher.",                                                                       // 8 — all kosher
    "But pork?",                                                                         // 9 — but pork
    "Forbidden.",                                                                         // 10 — forbidden
    "Shrimp and lobster?",                                                              // 11 — shrimp and lobster
    "Forbidden.",                                                                         // 12 — forbidden
    "We call it treif.",                                                                 // 13 — we call it treif
    "You must never mix meat and dairy. Separate plates. Separate everything.",           // 14 — you must never mix meat and dairy
    "Kosher is not a diet. It is discipline of the soul.",                                // 15 — kosher is not a diet
    "Follow for more Jewish wisdom.",                                                     // 16 — follow for more Jewish wisdom
  ],
});

// ── Rabbi angles ──
const rabbiWide = Image({
  model: varg.imageModel("nano-banana-pro/edit"),
  prompt: { text: `${CHAR} Wide shot. Same rabbi, warm kitchen, challah bread, golden light. Ultra sharp. 9:16.`, images: [RABBI_REF] },
  aspectRatio: "9:16",
});
const rabbiCloseup = Image({
  model: varg.imageModel("nano-banana-pro/edit"),
  prompt: { text: `${CHAR} Extreme close-up face. Warm amber side light. Wise eyes at camera. Dramatic shadows. Ultra sharp. 9:16.`, images: [rabbiWide, RABBI_REF] },
  aspectRatio: "9:16",
});
const rabbiKitchen = Image({
  model: varg.imageModel("nano-banana-pro/edit"),
  prompt: { text: `${CHAR} Medium shot warm kitchen table challah wine golden light smiling gently. Ultra sharp. 9:16.`, images: [rabbiWide, RABBI_REF] },
  aspectRatio: "9:16",
});

// ── VEED lipsync — rabbi talking clips ──
const lipsyncModel = varg.videoModel("veed-fabric-1.0");
const talk0  = Video({ model: lipsyncModel, keepAudio: true, prompt: { images: [rabbiWide],    audio: segments[0] } });
const talk1  = Video({ model: lipsyncModel, keepAudio: true, prompt: { images: [rabbiCloseup], audio: segments[1] } });
const talk2  = Video({ model: lipsyncModel, keepAudio: true, prompt: { images: [rabbiWide],    audio: segments[2] } });
const talk8  = Video({ model: lipsyncModel, keepAudio: true, prompt: { images: [rabbiCloseup], audio: segments[8] } });
const talk13 = Video({ model: lipsyncModel, keepAudio: true, prompt: { images: [rabbiCloseup], audio: segments[13] } });
const talk14 = Video({ model: lipsyncModel, keepAudio: true, prompt: { images: [rabbiWide],    audio: segments[14] } });
const talk15 = Video({ model: lipsyncModel, keepAudio: true, prompt: { images: [rabbiCloseup], audio: segments[15] } });
const talk16 = Video({ model: lipsyncModel, keepAudio: true, prompt: { images: [rabbiKitchen], audio: segments[16] } });

// ── Kosher food slideshow ──
const imgBeef    = Image({ model: varg.imageModel("nano-banana-pro"), prompt: "Beautiful food photography: raw beef steak on wooden cutting board, rosemary, warm kitchen light, shallow depth of field. Cinematic. 9:16.", aspectRatio: "9:16", zoom: "in" });
const imgChicken = Image({ model: varg.imageModel("nano-banana-pro"), prompt: "Whole roasted golden chicken on white ceramic plate, steam rising, herbs, rustic table, warm candlelight. Food editorial. 9:16.", aspectRatio: "9:16", zoom: "out" });
const imgSalmon  = Image({ model: varg.imageModel("nano-banana-pro"), prompt: "Fresh whole salmon fillet with glistening silver scales on crushed ice, lemon, dill sprigs. Dramatic top-down food photography. 9:16.", aspectRatio: "9:16", zoom: "in" });
const imgFruits  = Image({ model: varg.imageModel("nano-banana-pro"), prompt: "Vibrant arrangement of fresh fruits and vegetables — pomegranates cut open, figs, grapes, olives, dates — on dark wooden surface. Mediterranean feast. Warm golden light. 9:16.", aspectRatio: "9:16", zoom: "left" });
const imgEggs    = Image({ model: varg.imageModel("nano-banana-pro"), prompt: "Fresh brown eggs in rustic woven basket next to a golden braided challah bread on linen cloth. Warm morning light, kitchen background. Beautiful. 9:16.", aspectRatio: "9:16", zoom: "out" });

// ── Treif slideshow ──
const imgPork   = Image({ model: varg.imageModel("nano-banana-pro"), prompt: "Raw pork chop on dark cold slate surface. Blue-grey clinical lighting. Moody, cold, forbidden feel. 9:16.", aspectRatio: "9:16", zoom: "in" });
const imgShrimp = Image({ model: varg.imageModel("nano-banana-pro"), prompt: "Pile of raw shrimp and red lobster on dark ice. Cold blue lighting, harsh shadows. Clinical forbidden feel. 9:16.", aspectRatio: "9:16", zoom: "out" });

// ── Kitchen separation ──
const imgKitchen = Image({ model: varg.imageModel("nano-banana-pro"), prompt: "Overhead shot of kosher kitchen table split in two halves. Left: blue ceramic plates with dairy — cheese, milk, butter. Right: red ceramic plates with meat — chicken, beef. Clear dividing line in middle. Clean organized beautiful. Warm natural light. 9:16.", aspectRatio: "9:16", zoom: "out" });

// ── COMPOSE — clip durations from audio.duration ──
await render(
  <Render width={1080} height={1920} fps={30}>
    <Music model={varg.musicModel("music_v1")} prompt="gentle warm Jewish acoustic melody, soft guitar and light clarinet, friendly, contemplative" duration={40} volume={0.15} ducking />

    {/* Rabbi opening — lipsync talking clips */}
    <Clip duration={segments[0].duration}>{talk0}</Clip>
    <Clip duration={segments[1].duration}>{talk1}</Clip>
    <Clip duration={segments[2].duration}>{talk2}</Clip>

    {/* Kosher food slideshow — image + voiceover per food */}
    <Clip duration={segments[3].duration}>{imgBeef}{segments[3]}</Clip>
    <Clip duration={segments[4].duration}>{imgChicken}{segments[4]}</Clip>
    <Clip duration={segments[5].duration}>{imgSalmon}{segments[5]}</Clip>
    <Clip duration={segments[6].duration}>{imgFruits}{segments[6]}</Clip>
    <Clip duration={segments[7].duration}>{imgEggs}{segments[7]}</Clip>

    {/* Rabbi confirms: all kosher */}
    <Clip duration={segments[8].duration}>{talk8}</Clip>

    {/* Treif — cold, forbidden */}
    <Clip duration={segments[9].duration}>{imgPork}{segments[9]}</Clip>
    <Clip duration={segments[10].duration}>{segments[10]}</Clip>
    <Clip duration={segments[11].duration}>{imgShrimp}{segments[11]}</Clip>
    <Clip duration={segments[12].duration}>{segments[12]}</Clip>

    {/* Rabbi: we call it treif */}
    <Clip duration={segments[13].duration}>{talk13}</Clip>

    {/* Kitchen separation */}
    <Clip duration={segments[14].duration}>{imgKitchen}{segments[14]}</Clip>

    {/* Rabbi outro */}
    <Clip duration={segments[15].duration}>{talk15}</Clip>
    <Clip duration={segments[16].duration}>{talk16}</Clip>

    <Captions src={audio} style="tiktok" position="bottom" />
  </Render>,
  { output: "output/rabbi-kosher-full.mp4" }
);

console.log("Done! output/rabbi-kosher-full.mp4");

// ── OPTIONAL: Upload final video to varg storage ──
// Uncomment the block below to upload after render.
//
// const client = new VargClient({ apiKey: process.env.VARG_API_KEY! });
// const videoFile = Bun.file("output/rabbi-kosher-full.mp4");
// const blob = new Blob([await videoFile.arrayBuffer()], { type: "video/mp4" });
// const { url } = await client.uploadFile(blob, "video/mp4");
// console.log("Uploaded:", url);