Controlled Deployment for LLMInferenceService - Per-Participant Routes Architecture

controlled-deployment-overview.svg

<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 880 810" font-family="'Inter', 'Helvetica Neue', Arial, sans-serif">
  <defs>
    <marker id="arrow-blue" markerWidth="8" markerHeight="6" refX="8" refY="3" orient="auto">
      <path d="M0,0 L8,3 L0,6" fill="none" stroke="#4A7FBF" stroke-width="1.2"/>
    </marker>
    <marker id="arrow-blue-light" markerWidth="8" markerHeight="6" refX="8" refY="3" orient="auto">
      <path d="M0,0 L8,3 L0,6" fill="none" stroke="#9DC3E6" stroke-width="1"/>
    </marker>
    <marker id="arrow-purple" markerWidth="8" markerHeight="6" refX="8" refY="3" orient="auto">
      <path d="M0,0 L8,3 L0,6" fill="none" stroke="#B088C9" stroke-width="1"/>
    </marker>
    <marker id="arrow-traffic" markerWidth="10" markerHeight="7" refX="10" refY="3.5" orient="auto">
      <path d="M0,0 L10,3.5 L0,7" fill="#D97706" stroke="none"/>
    </marker>
    <filter id="shadow" x="-3%" y="-3%" width="106%" height="106%">
      <feDropShadow dx="0" dy="1" stdDeviation="2.5" flood-opacity="0.06"/>
    </filter>
    <!-- K8s-style resource icons -->
    <symbol id="ico-gw" viewBox="0 0 18 18">
      <rect width="18" height="18" rx="3" fill="#6B7280" opacity="0.1"/>
      <path d="M9,2 L15,5.5 L15,12.5 L9,16 L3,12.5 L3,5.5 Z" fill="none" stroke="#6B7280" stroke-width="1.2"/>
      <circle cx="9" cy="9" r="2" fill="#6B7280" opacity="0.4"/>
    </symbol>
    <symbol id="ico-route" viewBox="0 0 18 18">
      <rect width="18" height="18" rx="3" fill="#4A7FBF" opacity="0.1"/>
      <path d="M5,14 L5,7 Q5,4 9,4 Q13,4 13,7 L13,8" fill="none" stroke="#4A7FBF" stroke-width="1.3" stroke-linecap="round"/>
      <path d="M5,14 L5,10 Q5,8 8,7" fill="none" stroke="#4A7FBF" stroke-width="1.3" stroke-linecap="round" opacity="0.5"/>
      <circle cx="13" cy="10" r="1.5" fill="#4A7FBF"/>
      <circle cx="5" cy="14" r="1.5" fill="#4A7FBF"/>
    </symbol>
    <symbol id="ico-pool" viewBox="0 0 18 18">
      <rect width="18" height="18" rx="3" fill="#4A7FBF" opacity="0.1"/>
      <rect x="3" y="5" width="8" height="8" rx="1.5" fill="none" stroke="#4A7FBF" stroke-width="1"/>
      <rect x="6" y="3" width="8" height="8" rx="1.5" fill="none" stroke="#4A7FBF" stroke-width="1" opacity="0.6"/>
      <rect x="9" y="7" width="3" height="3" rx="0.5" fill="#4A7FBF" opacity="0.3"/>
    </symbol>
    <symbol id="ico-crd" viewBox="0 0 18 18">
      <rect width="18" height="18" rx="3" fill="#4A7FBF" opacity="0.1"/>
      <rect x="3" y="3" width="12" height="12" rx="2" fill="none" stroke="#4A7FBF" stroke-width="1"/>
      <path d="M7,3 L7,5 Q9,5.5 9,7 Q9,5.5 11,5 L11,3" fill="#4A7FBF" opacity="0.3" stroke="#4A7FBF" stroke-width="0.5"/>
      <text x="9" y="13" text-anchor="middle" font-size="6" font-weight="700" fill="#4A7FBF" font-family="sans-serif">CR</text>
    </symbol>
  </defs>

  <rect width="880" height="810" fill="#F6F8FA"/>

  <!-- Title -->
  <text x="440" y="32" text-anchor="middle" font-size="16" font-weight="700" fill="#1F2328" letter-spacing="-0.3">Controlled Deployment - Per-Participant Routes</text>

  <!-- ==================== GATEWAY ==================== -->
  <rect x="270" y="50" width="340" height="48" rx="8" fill="#fff" stroke="#D0D7DE" stroke-width="1.2" filter="url(#shadow)"/>
  <use href="#ico-gw" x="584" y="54" width="18" height="18"/>
  <text x="440" y="69" text-anchor="middle" font-size="10" fill="#8B949E" font-weight="500" letter-spacing="1.5">GATEWAY</text>
  <text x="440" y="87" text-anchor="middle" font-size="13" font-weight="600" fill="#1F2328">inference-gateway</text>

  <!-- Group boundary -->
  <rect x="20" y="108" width="840" height="660" rx="12" fill="#F0F6FF" stroke="#4A7FBF" stroke-width="1.2" stroke-dasharray="8,4" opacity="0.7"/>
  <rect x="34" y="100" width="160" height="20" rx="5" fill="#4A7FBF"/>
  <text x="114" y="114" text-anchor="middle" font-size="11" font-weight="600" fill="#fff">group: llama-70b</text>

  <!-- Traffic flow labels -->

  <!-- ==================== LAYER 1: ALL ARROWS (drawn first, behind everything) ==================== -->

  <!-- Traffic flow: Gateway -> HTTPRoutes -->
  <line x1="380" y1="98" x2="225" y2="142" stroke="#D97706" stroke-width="2" marker-end="url(#arrow-traffic)" opacity="0.35"/>
  <line x1="500" y1="98" x2="655" y2="142" stroke="#D97706" stroke-width="1.5" marker-end="url(#arrow-traffic)" opacity="0.15" stroke-dasharray="6,4"/>

  <!-- Owns arrows: LLMISVC -> HTTPRoute (L-shaped, outside edges) -->
  <path d="M 68 530 L 68 160 L 40 160" stroke="#8B949E" stroke-width="1" stroke-dasharray="2,3" opacity="0.5" fill="none"/>
  <rect x="49" y="340" width="38" height="16" rx="3" fill="#F0F6FF"/>
  <text x="68" y="351" text-anchor="middle" font-size="8" fill="#8B949E" font-weight="500">owns</text>
  <path d="M 832 530 L 832 160 L 840 160" stroke="#8B949E" stroke-width="1" stroke-dasharray="2,3" opacity="0.5" fill="none"/>
  <rect x="813" y="340" width="38" height="16" rx="3" fill="#F0F6FF"/>
  <text x="832" y="351" text-anchor="middle" font-size="8" fill="#8B949E" font-weight="500">owns</text>

  <!-- Cross-member arrows (drawn BEFORE InferencePool boxes so boxes cover overlap) -->
  <line x1="360" y1="307" x2="555" y2="403" stroke="#B088C9" stroke-width="1.2" stroke-dasharray="6,4" marker-end="url(#arrow-purple)" opacity="0.5"/>
  <line x1="520" y1="307" x2="325" y2="403" stroke="#B088C9" stroke-width="1.2" stroke-dasharray="6,4" marker-end="url(#arrow-purple)" opacity="0.5"/>

  <!-- Primary weighted arrows -->
  <line x1="225" y1="307" x2="225" y2="403" stroke="#4A7FBF" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
  <line x1="655" y1="307" x2="655" y2="403" stroke="#4A7FBF" stroke-width="1.5" marker-end="url(#arrow-blue)"/>

  <!-- ==================== LAYER 2: RESOURCE BOXES (drawn on top, covering arrow overlaps) ==================== -->

  <!-- HTTPRoute v1 -->
  <rect x="40" y="142" width="370" height="165" rx="8" fill="#fff" stroke="#4A7FBF" stroke-width="1.2" filter="url(#shadow)"/>
  <rect x="40" y="142" width="370" height="26" rx="8" fill="#4A7FBF" opacity="0.05"/>
  <use href="#ico-route" x="44" y="146" width="18" height="18"/>
  <text x="66" y="160" font-size="11" font-weight="700" fill="#4A7FBF">HTTPRoute</text>
  <text x="398" y="160" text-anchor="end" font-size="9" fill="#D97706" font-weight="500">active (oldest)</text>
  <text x="54" y="184" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">rules:</text>
  <text x="66" y="200" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#8B949E"># model-routing (weighted)</text>
  <text x="66" y="216" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">- header: X-Gateway-Model-Name</text>
  <text x="78" y="232" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#4A7FBF" font-weight="600">backendRefs: [v1: 9, v2: 1]</text>
  <text x="66" y="252" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#8B949E"># direct access (single)</text>
  <text x="66" y="268" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">- path: /ns/llama-70b-v1/...</text>
  <text x="78" y="284" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#4A7FBF" font-weight="600">backendRefs: [v1]</text>

  <!-- HTTPRoute v2 -->
  <rect x="470" y="142" width="370" height="165" rx="8" fill="#fff" stroke="#4A7FBF" stroke-width="1.2" filter="url(#shadow)"/>
  <rect x="470" y="142" width="370" height="26" rx="8" fill="#4A7FBF" opacity="0.05"/>
  <use href="#ico-route" x="474" y="146" width="18" height="18"/>
  <text x="496" y="160" font-size="11" font-weight="700" fill="#4A7FBF">HTTPRoute</text>
  <text x="828" y="160" text-anchor="end" font-size="9" fill="#8B949E">standby (newest)</text>
  <text x="484" y="184" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">rules:</text>
  <text x="496" y="200" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#8B949E"># model-routing (weighted)</text>
  <text x="496" y="216" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">- header: X-Gateway-Model-Name</text>
  <text x="508" y="232" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#4A7FBF" font-weight="600">backendRefs: [v1: 9, v2: 1]</text>
  <text x="496" y="252" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#8B949E"># direct access (single)</text>
  <text x="496" y="268" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">- path: /ns/llama-70b-v2/...</text>
  <text x="508" y="284" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#4A7FBF" font-weight="600">backendRefs: [v2]</text>

  <!-- Weight labels (on top of arrows) -->
  <rect x="200" y="345" width="50" height="20" rx="10" fill="#F0F6FF" stroke="#4A7FBF" stroke-width="1"/>
  <text x="225" y="359" text-anchor="middle" font-size="10" font-weight="700" fill="#4A7FBF">90%</text>
  <rect x="630" y="345" width="50" height="20" rx="10" fill="#F0F6FF" stroke="#4A7FBF" stroke-width="1"/>
  <text x="655" y="359" text-anchor="middle" font-size="10" font-weight="700" fill="#4A7FBF">10%</text>

  <!-- InferencePool v1 (drawn AFTER arrows to cover them) -->
  <rect x="80" y="403" width="280" height="52" rx="8" fill="#fff" stroke="#4A7FBF" stroke-width="1" filter="url(#shadow)"/>
  <rect x="80" y="403" width="280" height="26" rx="8" fill="#4A7FBF" opacity="0.05"/>
  <use href="#ico-pool" x="84" y="407" width="18" height="18"/>
  <text x="106" y="421" font-size="11" font-weight="600" fill="#4A7FBF">InferencePool</text>
  <text x="94" y="444" font-size="10" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">llama-70b-v1-inference-pool</text>

  <!-- InferencePool v2 (drawn AFTER arrows to cover them) -->
  <rect x="520" y="403" width="280" height="52" rx="8" fill="#fff" stroke="#4A7FBF" stroke-width="1" filter="url(#shadow)"/>
  <rect x="520" y="403" width="280" height="26" rx="8" fill="#4A7FBF" opacity="0.05"/>
  <use href="#ico-pool" x="524" y="407" width="18" height="18"/>
  <text x="546" y="421" font-size="11" font-weight="600" fill="#4A7FBF">InferencePool</text>
  <text x="534" y="444" font-size="10" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">llama-70b-v2-inference-pool</text>

  <!-- ==================== Pods v1 (centered on InferencePool v1: center=220) ==================== -->
  <rect x="120" y="466" width="200" height="36" rx="6" fill="#E8F0FE" stroke="#4A7FBF" stroke-width="0.8"/>
  <polygon points="175,475 186,481 186,491 175,497 164,491 164,481" fill="#fff" stroke="#4A7FBF" stroke-width="0.8"/>
  <text x="175" y="486" text-anchor="middle" dominant-baseline="central" font-size="7" fill="#4A7FBF" font-weight="600">P</text>
  <polygon points="207,475 218,481 218,491 207,497 196,491 196,481" fill="#fff" stroke="#4A7FBF" stroke-width="0.8"/>
  <text x="207" y="486" text-anchor="middle" dominant-baseline="central" font-size="7" fill="#4A7FBF" font-weight="600">P</text>
  <polygon points="239,475 250,481 250,491 239,497 228,491 228,481" fill="#fff" stroke="#4A7FBF" stroke-width="0.8"/>
  <text x="239" y="486" text-anchor="middle" dominant-baseline="central" font-size="7" fill="#4A7FBF" font-weight="600">P</text>
  <text x="305" y="488" text-anchor="end" font-size="9" fill="#8B949E">vllm:v2.1</text>

  <!-- ==================== Pods v2 (centered on InferencePool v2: center=660) ==================== -->
  <rect x="560" y="466" width="200" height="36" rx="6" fill="#E8F0FE" stroke="#4A7FBF" stroke-width="0.8"/>
  <polygon points="615,475 626,481 626,491 615,497 604,491 604,481" fill="#fff" stroke="#4A7FBF" stroke-width="0.8"/>
  <text x="615" y="486" text-anchor="middle" dominant-baseline="central" font-size="7" fill="#4A7FBF" font-weight="600">P</text>
  <polygon points="647,475 658,481 658,491 647,497 636,491 636,481" fill="#fff" stroke="#4A7FBF" stroke-width="0.8"/>
  <text x="647" y="486" text-anchor="middle" dominant-baseline="central" font-size="7" fill="#4A7FBF" font-weight="600">P</text>
  <polygon points="679,475 690,481 690,491 679,497 668,491 668,481" fill="#fff" stroke="#4A7FBF" stroke-width="0.8"/>
  <text x="679" y="486" text-anchor="middle" dominant-baseline="central" font-size="7" fill="#4A7FBF" font-weight="600">P</text>
  <text x="745" y="488" text-anchor="end" font-size="9" fill="#8B949E">vllm:v2.2</text>

  <!-- ==================== LLMISVCs (bottom corners, drawn last) ==================== -->

  <!-- LLMISVC v1 -->
  <rect x="60" y="530" width="350" height="200" rx="8" fill="#fff" stroke="#4A7FBF" stroke-width="1.3" filter="url(#shadow)"/>
  <rect x="60" y="530" width="350" height="26" rx="8" fill="#4A7FBF" opacity="0.07"/>
  <use href="#ico-crd" x="64" y="534" width="18" height="18"/>
  <text x="86" y="548" font-size="11" font-weight="700" fill="#4A7FBF">LLMInferenceService</text>
  <text x="398" y="548" text-anchor="end" font-size="10" fill="#8B949E" font-weight="500">llama-70b-v1</text>
  <text x="74" y="570" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">spec:</text>
  <text x="86" y="584" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">model:</text>
  <text x="98" y="598" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">name: llama-70b</text>
  <text x="98" y="612" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#C44D56" font-weight="600">uri: hf://meta-llama/Llama-3-70B</text>
  <text x="86" y="626" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#C44D56" font-weight="600">image: vllm/vllm:v2.1</text>
  <text x="86" y="640" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">router:</text>
  <text x="98" y="654" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">route:</text>
  <text x="110" y="668" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#4A7FBF" font-weight="700">group: llama-70b</text>
  <text x="110" y="682" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#C44D56" font-weight="700">weight: 9</text>
  <text x="110" y="696" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">http: {}</text>

  <!-- LLMISVC v2 -->
  <rect x="470" y="530" width="370" height="200" rx="8" fill="#fff" stroke="#4A7FBF" stroke-width="1.3" filter="url(#shadow)"/>
  <rect x="470" y="530" width="370" height="26" rx="8" fill="#4A7FBF" opacity="0.07"/>
  <use href="#ico-crd" x="474" y="534" width="18" height="18"/>
  <text x="496" y="548" font-size="11" font-weight="700" fill="#4A7FBF">LLMInferenceService</text>
  <text x="828" y="548" text-anchor="end" font-size="10" fill="#8B949E" font-weight="500">llama-70b-v2</text>
  <text x="484" y="570" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">spec:</text>
  <text x="496" y="584" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">model:</text>
  <text x="508" y="598" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">name: llama-70b</text>
  <text x="508" y="612" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#C44D56" font-weight="600">uri: hf://meta-llama/Llama-3.1-70B</text>
  <text x="496" y="626" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#C44D56" font-weight="600">image: vllm/vllm:v2.2</text>
  <text x="496" y="640" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">router:</text>
  <text x="508" y="654" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">route:</text>
  <text x="520" y="668" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#4A7FBF" font-weight="700">group: llama-70b</text>
  <text x="520" y="682" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#C44D56" font-weight="700">weight: 1</text>
  <text x="520" y="696" font-size="10.5" font-family="'JetBrains Mono', 'Fira Code', monospace" fill="#57606A">http: {}</text>

  <!-- ==================== LEGEND ==================== -->
  <rect x="20" y="775" width="840" height="32" rx="6" fill="#fff" stroke="#D0D7DE" stroke-width="0.8"/>
  <path d="M 36 792 L 70 792" stroke="#D97706" stroke-width="2" opacity="0.4"/>
  <text x="78" y="796" font-size="10" fill="#57606A">Traffic</text>
  <line x1="118" y1="792" x2="152" y2="792" stroke="#4A7FBF" stroke-width="1.5"/>
  <text x="160" y="796" font-size="10" fill="#57606A">Weighted ref</text>
  <line x1="238" y1="792" x2="272" y2="792" stroke="#B088C9" stroke-width="1.2" stroke-dasharray="6,4" opacity="0.6"/>
  <text x="280" y="796" font-size="10" fill="#57606A">Cross-member ref</text>
  <line x1="405" y1="792" x2="439" y2="792" stroke="#8B949E" stroke-width="1" stroke-dasharray="2,3" opacity="0.6"/>
  <text x="447" y="796" font-size="10" fill="#57606A">OwnerRef</text>
  <text x="510" y="796" font-size="10" fill="#C44D56" font-weight="600">red</text>
  <text x="672" y="796" font-size="10" fill="#57606A">= differs between versions</text>
</svg>