frame transformer with readPixels

webgl-transformer-readpixels.js

const W = 1280; 
const H = 720; 

async function start() {
  const buffer = new Uint8Array(W * H * 1.5);
  const bufferRGB = new Uint8Array(W * H * 4);

  const transformer = new TransformStream({
    async transform(videoFrame, controller) {
      const startTs = performance.now();
      await videoFrame.copyTo(buffer);

      const { gl } = glRef;   // get the webGL reference 

      gl.activeTexture(gl.TEXTURE1);   // Upload the image into the texture.
      gl.texImage2D(
        gl.TEXTURE_2D,
        0,
        gl.LUMINANCE,
        W,
        H * 1.5,
        0,
        gl.LUMINANCE,
        gl.UNSIGNED_BYTE,
        buffer,
        0
      );

      gl.drawArrays(gl.TRIANGLES, 0, 6);       // Draw the image

      gl.flush();
      gl.finish();

      gl.readPixels(0, 0, W, H, gl.RGBA, gl.UNSIGNED_BYTE, bufferRGB);

      const init = {
        timestamp: videoFrame.timestamp,
        codedWidth: W,
        codedHeight: H,
        format: "RGBA",
      };
      const newFrame = new VideoFrame(bufferRGB, init);

      videoFrame.close();
      controller.enqueue(newFrame);
    }
  });

  trackProcessor.readable
    .pipeThrough(transformer)
    .pipeTo(trackGenerator.writable);

  const streamAfter = new MediaStream([trackGenerator]);
}