MediaPipe - Holistic

index.html

  <div class="container">
    <video class="input_video"></video>
    <div class="canvas-container">
    <canvas class="output_canvas" width="1280px" height="720px">
    </canvas>
    </div>
    <div class="loading">
      <div class="spinner"></div>
      <div class="message">
        Loading
      </div>
    </div>
    <a class="abs logo" href="http://www.mediapipe.dev" target="_blank">
      <div style="display: flex;align-items: center;bottom: 0;right: 10px;">
        <img class="logo" src="logo_white.png" alt="" style="
          height: 50px;">
        <span class="title">MediaPipe</span>
      </div>
    </a>
    <div class="shoutout">
      <div>
        <a href="https://solutions.mediapipe.dev/holistic">
          Click here for more info
        </a>
      </div>
    </div>
  </div>
  <div class="control-panel">
  </div>

mediapipe-holistic.markdown

MediaPipe - Holistic

A Pen by JaT5 on CodePen.

License.

script.typescript

import DeviceDetector from "https://cdn.skypack.dev/[email protected]";

// Usage: testSupport({client?: string, os?: string}[])
// Client and os are regular expressions.
// See: https://cdn.jsdelivr.net/npm/[email protected]/README.md for
// legal values for client and os
testSupport([
  {client: 'Chrome'},
]);

function testSupport(supportedDevices:{client?: string; os?: string;}[]) {
  const deviceDetector = new DeviceDetector();
  const detectedDevice = deviceDetector.parse(navigator.userAgent);

  let isSupported = false;
  for (const device of supportedDevices) {
    if (device.client !== undefined) {
      const re = new RegExp(`^${device.client}$`);
      if (!re.test(detectedDevice.client.name)) {
        continue;
      }
    }
    if (device.os !== undefined) {
      const re = new RegExp(`^${device.os}$`);
      if (!re.test(detectedDevice.os.name)) {
        continue;
      }
    }
    isSupported = true;
    break;
  }
  if (!isSupported) {
    alert(`This demo, running on ${detectedDevice.client.name}/${detectedDevice.os.name}, ` +
          `is not well supported at this time, continue at your own risk.`);
  }
}

const controls = window;
const mpHolistic = window;
const drawingUtils = window;

const config = {locateFile: (file) => {
  return `https://cdn.jsdelivr.net/npm/@mediapipe/holistic@` +
         `${mpHolistic.VERSION}/${file}`;
}};

// Our input frames will come from here.
const videoElement =
    document.getElementsByClassName('input_video')[0] as HTMLVideoElement;
const canvasElement =
    document.getElementsByClassName('output_canvas')[0] as HTMLCanvasElement;
const controlsElement =
    document.getElementsByClassName('control-panel')[0] as HTMLDivElement;
const canvasCtx = canvasElement.getContext('2d')!;

// We'll add this to our control panel later, but we'll save it here so we can
// call tick() each time the graph runs.
const fpsControl = new controls.FPS();

// Optimization: Turn off animated spinner after its hiding animation is done.
const spinner = document.querySelector('.loading')! as HTMLDivElement;
spinner.ontransitionend = () => {
  spinner.style.display = 'none';
};

function removeElements(
    landmarks: mpHolistic.NormalizedLandmarkList, elements: number[]) {
  for (const element of elements) {
    delete landmarks[element];
  }
}

function removeLandmarks(results: mpHolistic.Results) {
  if (results.poseLandmarks) {
    removeElements(
        results.poseLandmarks,
        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17, 18, 19, 20, 21, 22]);
  }
}

function connect(
    ctx: CanvasRenderingContext2D,
    connectors:
        Array<[mpHolistic.NormalizedLandmark, mpHolistic.NormalizedLandmark]>):
    void {
  const canvas = ctx.canvas;
  for (const connector of connectors) {
    const from = connector[0];
    const to = connector[1];
    if (from && to) {
      if (from.visibility && to.visibility &&
          (from.visibility < 0.1 || to.visibility < 0.1)) {
        continue;
      }
      ctx.beginPath();
      ctx.moveTo(from.x * canvas.width, from.y * canvas.height);
      ctx.lineTo(to.x * canvas.width, to.y * canvas.height);
      ctx.stroke();
    }
  }
}

let activeEffect = 'mask';
function onResults(results: mpHolistic.Results): void {
  // Hide the spinner.
  document.body.classList.add('loaded');

  // Remove landmarks we don't want to draw.
  removeLandmarks(results);

  // Update the frame rate.
  fpsControl.tick();

  // Draw the overlays.
  canvasCtx.save();
  canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);

  if (results.segmentationMask) {
    canvasCtx.drawImage(
        results.segmentationMask, 0, 0, canvasElement.width,
        canvasElement.height);

    // Only overwrite existing pixels.
    if (activeEffect === 'mask' || activeEffect === 'both') {
      canvasCtx.globalCompositeOperation = 'source-in';
      // This can be a color or a texture or whatever...
      canvasCtx.fillStyle = '#00FF007F';
      canvasCtx.fillRect(0, 0, canvasElement.width, canvasElement.height);
    } else {
      canvasCtx.globalCompositeOperation = 'source-out';
      canvasCtx.fillStyle = '#0000FF7F';
      canvasCtx.fillRect(0, 0, canvasElement.width, canvasElement.height);
    }

    // Only overwrite missing pixels.
    canvasCtx.globalCompositeOperation = 'destination-atop';
    canvasCtx.drawImage(
        results.image, 0, 0, canvasElement.width, canvasElement.height);

    canvasCtx.globalCompositeOperation = 'source-over';
  } else {
     canvasCtx.drawImage(
         results.image, 0, 0, canvasElement.width, canvasElement.height);
  }

  // Connect elbows to hands. Do this first so that the other graphics will draw
  // on top of these marks.
  canvasCtx.lineWidth = 5;
  if (results.poseLandmarks) {
    if (results.rightHandLandmarks) {
      canvasCtx.strokeStyle = 'white';
      connect(canvasCtx, [[
                results.poseLandmarks[mpHolistic.POSE_LANDMARKS.RIGHT_ELBOW],
                results.rightHandLandmarks[0]
              ]]);
    }
    if (results.leftHandLandmarks) {
      canvasCtx.strokeStyle = 'white';
      connect(canvasCtx, [[
                results.poseLandmarks[mpHolistic.POSE_LANDMARKS.LEFT_ELBOW],
                results.leftHandLandmarks[0]
              ]]);
    }
  }

  // Pose...
  drawingUtils.drawConnectors(
      canvasCtx, results.poseLandmarks, mpHolistic.POSE_CONNECTIONS,
      {color: 'white'});
  drawingUtils.drawLandmarks(
      canvasCtx,
      Object.values(mpHolistic.POSE_LANDMARKS_LEFT)
          .map(index => results.poseLandmarks[index]),
      {visibilityMin: 0.65, color: 'white', fillColor: 'rgb(255,138,0)'});
  drawingUtils.drawLandmarks(
      canvasCtx,
      Object.values(mpHolistic.POSE_LANDMARKS_RIGHT)
          .map(index => results.poseLandmarks[index]),
      {visibilityMin: 0.65, color: 'white', fillColor: 'rgb(0,217,231)'});

  // Hands...
  drawingUtils.drawConnectors(
      canvasCtx, results.rightHandLandmarks, mpHolistic.HAND_CONNECTIONS,
      {color: 'white'});
  drawingUtils.drawLandmarks(canvasCtx, results.rightHandLandmarks, {
    color: 'white',
    fillColor: 'rgb(0,217,231)',
    lineWidth: 2,
    radius: (data: drawingUtils.Data) => {
      return drawingUtils.lerp(data.from!.z!, -0.15, .1, 10, 1);
    }
  });
  drawingUtils.drawConnectors(
      canvasCtx, results.leftHandLandmarks, mpHolistic.HAND_CONNECTIONS,
      {color: 'white'});
  drawingUtils.drawLandmarks(canvasCtx, results.leftHandLandmarks, {
    color: 'white',
    fillColor: 'rgb(255,138,0)',
    lineWidth: 2,
    radius: (data: drawingUtils.Data) => {
      return drawingUtils.lerp(data.from!.z!, -0.15, .1, 10, 1);
    }
  });

  // Face...
  drawingUtils.drawConnectors(
      canvasCtx, results.faceLandmarks, mpHolistic.FACEMESH_TESSELATION,
      {color: '#C0C0C070', lineWidth: 1});
  drawingUtils.drawConnectors(
      canvasCtx, results.faceLandmarks, mpHolistic.FACEMESH_RIGHT_EYE,
      {color: 'rgb(0,217,231)'});
  drawingUtils.drawConnectors(
      canvasCtx, results.faceLandmarks, mpHolistic.FACEMESH_RIGHT_EYEBROW,
      {color: 'rgb(0,217,231)'});
  drawingUtils.drawConnectors(
      canvasCtx, results.faceLandmarks, mpHolistic.FACEMESH_LEFT_EYE,
      {color: 'rgb(255,138,0)'});
  drawingUtils.drawConnectors(
      canvasCtx, results.faceLandmarks, mpHolistic.FACEMESH_LEFT_EYEBROW,
      {color: 'rgb(255,138,0)'});
  drawingUtils.drawConnectors(
      canvasCtx, results.faceLandmarks, mpHolistic.FACEMESH_FACE_OVAL,
      {color: '#E0E0E0', lineWidth: 5});
  drawingUtils.drawConnectors(
      canvasCtx, results.faceLandmarks, mpHolistic.FACEMESH_LIPS,
      {color: '#E0E0E0', lineWidth: 5});

  canvasCtx.restore();
}

const holistic = new mpHolistic.Holistic(config);
holistic.onResults(onResults);

// Present a control panel through which the user can manipulate the solution
// options.
new controls
    .ControlPanel(controlsElement, {
      selfieMode: true,
      modelComplexity: 1,
      smoothLandmarks: true,
      enableSegmentation: false,
      smoothSegmentation: true,
      minDetectionConfidence: 0.5,
      minTrackingConfidence: 0.5,
      effect: 'background',
    })
    .add([
      new controls.StaticText({title: 'MediaPipe Holistic'}),
      fpsControl,
      new controls.Toggle({title: 'Selfie Mode', field: 'selfieMode'}),
      new controls.SourcePicker({
        onSourceChanged: () => {
          // Resets because the pose gives better results when reset between
          // source changes.
          holistic.reset();
        },
        onFrame:
            async (input: controls.InputImage, size: controls.Rectangle) => {
              const aspect = size.height / size.width;
              let width: number, height: number;
              if (window.innerWidth > window.innerHeight) {
                height = window.innerHeight;
                width = height / aspect;
              } else {
                width = window.innerWidth;
                height = width * aspect;
              }
              canvasElement.width = width;
              canvasElement.height = height;
              await holistic.send({image: input});
            },
      }),
      new controls.Slider({
        title: 'Model Complexity',
        field: 'modelComplexity',
        discrete: ['Lite', 'Full', 'Heavy'],
      }),
      new controls.Toggle(
          {title: 'Smooth Landmarks', field: 'smoothLandmarks'}),
      new controls.Toggle(
          {title: 'Enable Segmentation', field: 'enableSegmentation'}),
      new controls.Toggle(
          {title: 'Smooth Segmentation', field: 'smoothSegmentation'}),
      new controls.Slider({
        title: 'Min Detection Confidence',
        field: 'minDetectionConfidence',
        range: [0, 1],
        step: 0.01
      }),
      new controls.Slider({
        title: 'Min Tracking Confidence',
        field: 'minTrackingConfidence',
        range: [0, 1],
        step: 0.01
      }),
      new controls.Slider({
        title: 'Effect',
        field: 'effect',
        discrete: {'background': 'Background', 'mask': 'Foreground'},
      }),
    ])
    .on(x => {
      const options = x as mpHolistic.Options;
      videoElement.classList.toggle('selfie', options.selfieMode);
      activeEffect = (x as {[key: string]: string})['effect'];
      holistic.setOptions(options);
    });

style.scss

@keyframes spin {
  0% {
    transform: rotate(0deg);
  }
  100% {
    transform: rotate(360deg);
  }
}

.abs {
  position: absolute;
}

a {
  color: white;
  text-decoration: none;
  &:hover {
    color: lightblue;
  }
}

body {
  bottom: 0;
  font-family: 'Titillium Web', sans-serif;
  color: white;
  left: 0;
  margin: 0;
  position: absolute;
  right: 0;
  top: 0;
  transform-origin: 0px 0px;
  overflow: hidden;
}

.container {
  position: absolute;
  background-color: #596e73;
  width: 100%;
  max-height: 100%;
}

.input_video {
  display: none;
  position: absolute;
  top: 0;
  left: 0;
  right: 0;
  bottom: 0;
  &.selfie {
    transform: scale(-1, 1);
  }
}

.input_image {
  position: absolute;
}

.canvas-container {
  display:flex;
  height: 100%;
  width: 100%;
  justify-content: center;
  align-items:center;
}

.output_canvas {
  max-width: 100%;
  display: block;
  position: relative;
  left: 0;
  top: 0;
}

.logo {
  bottom: 10px;
  right: 20px;

  .title {
    color: white;
    font-size: 28px;
  }

  .subtitle {
    position: relative;
    color: white;
    font-size: 10px;
    left: -30px;
    top: 20px;
  }
}

.control-panel {
  position: absolute;
  left: 10px;
  top: 10px;
}

.loading {
  display: flex;
  position: absolute;
  top: 0;
  right: 0;
  bottom: 0;
  left: 0;
  align-items: center;
  backface-visibility: hidden;
  justify-content: center;
  opacity: 1;
  transition: opacity 1s;

  .message {
    font-size: x-large;
  }

  .spinner {
    position: absolute;
    width: 120px;
    height: 120px;
    animation: spin 1s linear infinite;
    border: 32px solid #bebebe;
    border-top: 32px solid #3498db;
    border-radius: 50%;
  }
}

.loaded .loading {
  opacity: 0;
}

.shoutout {
  left: 0;
  right: 0;
  bottom: 40px;
  text-align: center;
  font-size: 24px;
  position: absolute;
}