A photo of Max Glenister Max Glenister

10: HTML5 Video

Today we’re looking at HTML5 video – well not in the traditional sense. We’re looking at using the <video> element to create a texture and then applying it to an object in our 3D scene.

I’m using the trailer for the CC-Licensed “Big Buck Bunny” movie for my video.

Creating a little cinema

Before we get on to actually using a video, we need somewhere suitable to play it. So let’s build a cinema.

Abstractly a cinema is a bunch of seats facing a screen. So let’s work with that. First the chairs layout.

A chair is made of a number of BoxGeometry’s grouped inside an Object3D:

var chairObject = new THREE.Object3D();
var chairBase = core.build(
  'BoxGeometry'
  // etc...
);
var chairBack = core.build(
  'BoxGeometry'
  // etc...
);

chairObject.add(chairBase);
chairObject.add(chairBack);
// etc.

I’ve not given the full exhaustive source for creating a chair there, but you can see how I do it in full in 10/demo.js#L40-L97.

We can define the layout of the chairs as a multidimenstional array, with 0 meaning no chair and 1 meaning a chair:

var chairArrangement = [
  [0, 0, 1, 1, 0, 1, 1, 0, 0],
  [0, 1, 1, 1, 0, 1, 1, 1, 0],
  [1, 1, 1, 1, 0, 1, 1, 1, 1],
  [1, 1, 1, 1, 0, 1, 1, 1, 1],
];

And then we can loop through this array and position each chair model in the scene:

var height = 0;
var metrics = {
  width: 6,
  height: 6,
  depth: 8,
};
var chairs = [];
var xOffset = (metrics.width * chairArrangement[0].length) / 2;
chairArrangement.forEach(function(row, i) { // for each row
  var _row = [];
  row.forEach(function(col, j) { // for each col
    if (col == 1) {
      // create a clone of our chair
      var chair = chairObject.clone();
      // position the chair
      chair.position.x = (j * metrics.width) - xOffset;
      chair.position.y = height * (0.3 * metrics.height);
      chair.position.z = i * metrics.depth;

      // add the chair to the scene
      scene.add(chair);
      _row[j] = chair;
    }
  });
  chairs.push(_row);
  height++;
});

Now that we’ve made a bunch of chairs, wouldn’t it be cool to randomly position the camera in one on each load? It’s quite simple to do, just take the chair.position and use it to set the camera.position (with some minor tweaking)

var randomIndex = function (array) {
  return Math.floor(Math.random() * array.length);
};
var randomSeat = function () {
  var indexR = randomIndex(chairArrangement);
  var indexC = randomIndex(chairArrangement[indexR]);

  // Make sure we've got a chair (1) not an empty space (0)
  while (chairArrangement[indexR][indexC] == 0) {
    indexC = randomIndex(chairArrangement[indexR]);
  }
  return {
    row: indexR,
    col: indexC
  };
};

var gotochair = function (seat) {
  var targetChair = chairs[seat.row][seat.col];
  camera.position.set(
    targetChair.position.x,
    targetChair.position.y + 4,
    targetChair.position.z
  );
  camera.lookAt(
    new T.Vector3(
      screen.position.x,
      screen.position.y,
      screen.position.z
    )
  );
};

gotochair(randomSeat());

The screen

For the screen we just have a massive BoxGeometry again, positioned in front of the rows of chairs. We’ll texture this using the video later on, but for an initial state it’s a good place to put some instructions.


var videotexture = textureLoader.load('video.png');
var screen = core.build(
  'BoxGeometry', [120, 60, 0],
  'MeshLambertMaterial', [{
    color: 0xffffff,
    map: videotexture,
    overdraw: 0.5
  }]
);
screen.position.set(0, 20, -50);
scene.add(screen);

And now that we have a screen, we can load a video. For this we’re going to create a video element, with a number of source element children that specify the various formats to use.

// Create a video element
var video = document.createElement('video');
  video.poster = 'video.png';
// Add the sources...
// webm
var videoSource = document.createElement('source');
videoSource.src = 'video.webm';
videoSource.type = 'video/webm';
video.appendChild(videoSource);
// ogg
videoSource = document.createElement('source');
videoSource.src = 'video.ogv';
videoSource.type = 'video/ogg';
video.appendChild(videoSource);
// mp4
videoSource = document.createElement('source');
videoSource.src = 'video.mp4';
videoSource.type = 'video/mp4';
video.appendChild(videoSource);

Next we’ll create a 2d canvas on which we’ll later load the video frames:

// Make your video canvas
var videocanvas = document.createElement('canvas');
var videocanvasctx = videocanvas.getContext('2d');

// Set its size (multiples of 8!)
var vidw = 1024;
var vidh = 512;

videocanvas.width = vidw;
videocanvas.height = vidh;

videocanvasctx.fillStyle = "#222222";
videocanvasctx.fillRect(0,0,vidw,vidh);

Then when the video is playing, update the canvas in our render loop so that it gets the current frame of the video:

var update = function () {
  if ( video.readyState === video.HAVE_ENOUGH_DATA ){
    //draw video to canvas starting from upper left corner
    videocanvasctx.drawImage(video, 0, 0);
    //tell texture object it needs to be updated
    videotexture.needsUpdate = true;
  }
  // Other bits in our loop...
};

To trigger the video.load() and video.play() and assign the texture to our screen, we’re using vreticle.js to provide an ongazelong event.

var playing = false;
screen.ongazelong = function () {
  if (!playing) {
    videotexture = new THREE.Texture(videocanvas);
    videotexture.wrapS = videotexture.wrapT = T.RepeatWrapping;
    videotexture.repeat.x = 1;
    videotexture.repeat.y = 1;
    videotexture.offset.x = -.15;
    videotexture.offset.y = .1;
    screen.material.map = videotexture;
    video.load();
    video.play();
    reticle.remove_collider(screen);
  }
};
reticle.add_collider(screen);

Issues on mobile

Initially I had tried to simply video.load() and video.play() the video when the scene loaded. This worked fine on Chrome desktop, but not on Chrome mobile. After a lot of fiddling and Googling I found that this is because Chrome limits automatically triggering video playback with JavaScript. For example you don’t want a website you’re viewing on your phone to automatically load and play a massive video file over your data connection.

That’s fair enough, so to get around this we have to trigger the video.load() and video.play() from within a click/touchstart event.

Ammending our earlier screen.ongazelong we can just add a check to see if we’re on mobile, and then addEventListener to listen for a click and trigger video.load() and video.play(). You can see the updated screen.ongazelong even in demo.js#L195-L229

By this point the video is playing nicely (not so performantly on mobile though!) but I wanted to add another feature. When you’re sat in a cinema or even when you’re watching a TV, the light from the screen is usually cast around the room. So let’s do that in our scene.

I’m using a library called Color Thief to extract the four dominant colors in each frame of the video, and then using these to color four THREE.PointLight’s that I’ve positioned around the screen.

First setup the lights:

var lightsobj = [
  [0xffffff, -25, 30, -35],
  [0xffffff, 25, 30, -35],
  [0xffffff, -25, 0, -35],
  [0xffffff, 25, 0, -35]
];
var lights = [];
lightsobj.forEach(function (lobj, i) {
  var _light = new T.PointLight(lobj[0], .5, 0, 1);
  _light.position.set(lobj[1], lobj[2], lobj[3]);
  lights.push(_light);
  scene.add(_light);
});

And then later on in the render loop, each time we get a new frame we can grab the dominant colors and recolor the lights:

var colorThief = new ColorThief();

var update = function () {
  // Other video frame stuff...

  var domcolors = colorThief.getPalette(videocanvas, lightsobj.length);
  domcolors.forEach(function (color, i) {
    var colorrgb = core.template("rgb({r}, {g}, {b})",{
      r:color[0],
      g:color[1],
      b:color[2],
    });
    var tcolor = new THREE.Color(colorrgb);
    lights[i].color = tcolor;
  });

  // Other render loop stuff
}

And we’re done. You might want to mess with the intensity of the lights, or play with having more lights/dominant color samples.

A note about the delay in publishing this write-up

So this write-up (though dated 10/10/16) was actually published on 11/10/16. I decided to refactor most of my core.js library while working on day 10’s hack and it took a little longer than I had planned.

Cardboctober

View this Cardboctober hack 10: HTML5 Video

View the other submissions for day 10 on the Cardboctober website.

Check out my other Cardoctober posts here: /cardboctober

About the author

A photo of Max Glenister

Max Glenister is a Front-end Developer based in Oxfordshire. For work he spends his time designing, validating and implementing user interfaces. For fun he tinkers with Virtual Reality, 3D printing, embedded systems, game development and many other things.

You can keep up with Max on Github, Twitter and Reddit