Convert between screen and world coordinates in isometric 3D

Question

I'm working on a a simple 3D scene in WebGL. The purpose is to:

1. Implement an isometric viewing angle in orthographic 3D
2. Translate between XY screen coordinates and XY world coordinates

Codepen: https://codepen.io/Candleout/pen/RwvxOJy

Sometimes when I load the page the scene appears blank or is not rendered correctly. Not sure why this happens. If this happens to you, refresh the page and/or scene it and it should hopefully work (x = 500 and y = -150 at screen center).

In the scene, X (red axis) represent East-West and Y (green axis) represents North-South. X increases to the East and Y increases to the North. Z (blue axis) represents the vertical axis and increases with elevation.

Camera setup:

cameraPosition = [x + 1, y - 1, 1]
cameraTarget = [x, y, 0]
up = [0, 0, 1]

With this setup, the objects are correctly positioned (or so it appears) in three dimensions. But how do you convert between screen coords and world coords?

Below is my current (failed) attempt to converting from screen space to world space. It includes the following steps:

• Normalize the screen coordinates
• Invert the view projection matrix
• Apply the inverted matrix to the normalized screen coordinates

With this code, only the coordinate in the center of the screen is turning out correct (500, -150). Everything else is skewed in some way. If you follow the y axis with the mouse, the x values will update as well, and vice versa.

If anyone can tell me why this is happening and what I should be doing instead it would be very much appreciated.

The world coordinates are visible in the bottom left corner of the Codepen.

function screenToWorld(screenCoords, viewProjectionMatrix, screenWidth, screenHeight) {
  // Convert screen coords into clip coords
  const ndcX = (screenCoords[0] / screenWidth) * 2 - 1;
  const ndcY = 1 - (screenCoords[1] / screenHeight) * 2;
  let clipCoords = [ndcX, ndcY, 0, 1];

  // Invert view projection matrix
  const inverseViewProjectionMatrix = m4.inverse(viewProjectionMatrix);

  // Apply inverted matrix to normalized screen coords
  let worldCoords = m4.transformMat4([], clipCoords, inverseViewProjectionMatrix);

  return worldCoords.slice(0, 3);
}

function inverse(m) {
    let m00 = m[0 * 4 + 0];
    let m01 = m[0 * 4 + 1];
    let m02 = m[0 * 4 + 2];
    let m03 = m[0 * 4 + 3];
    let m10 = m[1 * 4 + 0];
    let m11 = m[1 * 4 + 1];
    let m12 = m[1 * 4 + 2];
    let m13 = m[1 * 4 + 3];
    let m20 = m[2 * 4 + 0];
    let m21 = m[2 * 4 + 1];
    let m22 = m[2 * 4 + 2];
    let m23 = m[2 * 4 + 3];
    let m30 = m[3 * 4 + 0];
    let m31 = m[3 * 4 + 1];
    let m32 = m[3 * 4 + 2];
    let m33 = m[3 * 4 + 3];
    let tmp_0  = m22 * m33;
    let tmp_1  = m32 * m23;
    let tmp_2  = m12 * m33;
    let tmp_3  = m32 * m13;
    let tmp_4  = m12 * m23;
    let tmp_5  = m22 * m13;
    let tmp_6  = m02 * m33;
    let tmp_7  = m32 * m03;
    let tmp_8  = m02 * m23;
    let tmp_9  = m22 * m03;
    let tmp_10 = m02 * m13;
    let tmp_11 = m12 * m03;
    let tmp_12 = m20 * m31;
    let tmp_13 = m30 * m21;
    let tmp_14 = m10 * m31;
    let tmp_15 = m30 * m11;
    let tmp_16 = m10 * m21;
    let tmp_17 = m20 * m11;
    let tmp_18 = m00 * m31;
    let tmp_19 = m30 * m01;
    let tmp_20 = m00 * m21;
    let tmp_21 = m20 * m01;
    let tmp_22 = m00 * m11;
    let tmp_23 = m10 * m01;

    let t0 = (tmp_0 * m11 + tmp_3 * m21 + tmp_4 * m31) -
             (tmp_1 * m11 + tmp_2 * m21 + tmp_5 * m31);
    let t1 = (tmp_1 * m01 + tmp_6 * m21 + tmp_9 * m31) -
             (tmp_0 * m01 + tmp_7 * m21 + tmp_8 * m31);
    let t2 = (tmp_2 * m01 + tmp_7 * m11 + tmp_10 * m31) -
             (tmp_3 * m01 + tmp_6 * m11 + tmp_11 * m31);
    let t3 = (tmp_5 * m01 + tmp_8 * m11 + tmp_11 * m21) -
             (tmp_4 * m01 + tmp_9 * m11 + tmp_10 * m21);

    let d = 1.0 / (m00 * t0 + m10 * t1 + m20 * t2 + m30 * t3);

    return [
      d * t0,
      d * t1,
      d * t2,
      d * t3,
      d * ((tmp_1 * m10 + tmp_2 * m20 + tmp_5 * m30) -
           (tmp_0 * m10 + tmp_3 * m20 + tmp_4 * m30)),
      d * ((tmp_0 * m00 + tmp_7 * m20 + tmp_8 * m30) -
           (tmp_1 * m00 + tmp_6 * m20 + tmp_9 * m30)),
      d * ((tmp_3 * m00 + tmp_6 * m10 + tmp_11 * m30) -
           (tmp_2 * m00 + tmp_7 * m10 + tmp_10 * m30)),
      d * ((tmp_4 * m00 + tmp_9 * m10 + tmp_10 * m20) -
           (tmp_5 * m00 + tmp_8 * m10 + tmp_11 * m20)),
      d * ((tmp_12 * m13 + tmp_15 * m23 + tmp_16 * m33) -
           (tmp_13 * m13 + tmp_14 * m23 + tmp_17 * m33)),
      d * ((tmp_13 * m03 + tmp_18 * m23 + tmp_21 * m33) -
           (tmp_12 * m03 + tmp_19 * m23 + tmp_20 * m33)),
      d * ((tmp_14 * m03 + tmp_19 * m13 + tmp_22 * m33) -
           (tmp_15 * m03 + tmp_18 * m13 + tmp_23 * m33)),
      d * ((tmp_17 * m03 + tmp_20 * m13 + tmp_23 * m23) -
           (tmp_16 * m03 + tmp_21 * m13 + tmp_22 * m23)),
      d * ((tmp_14 * m22 + tmp_17 * m32 + tmp_13 * m12) -
           (tmp_16 * m32 + tmp_12 * m12 + tmp_15 * m22)),
      d * ((tmp_20 * m32 + tmp_12 * m02 + tmp_19 * m22) -
           (tmp_18 * m22 + tmp_21 * m32 + tmp_13 * m02)),
      d * ((tmp_18 * m12 + tmp_23 * m32 + tmp_15 * m02) -
           (tmp_22 * m32 + tmp_14 * m02 + tmp_19 * m12)),
      d * ((tmp_22 * m22 + tmp_16 * m02 + tmp_21 * m12) -
           (tmp_20 * m12 + tmp_23 * m22 + tmp_17 * m02)),
    ];
  }

function transformMat4(out, vec, mat) {
    const x = vec[0];
    const y = vec[1];
    const z = vec[2] || 0;
    const w = vec[3] || 1;
  
    out[0] = mat[0] * x + mat[4] * y + mat[8] * z + mat[12] * w;
    out[1] = mat[1] * x + mat[5] * y + mat[9] * z + mat[13] * w;
    out[2] = mat[2] * x + mat[6] * y + mat[10] * z + mat[14] * w;
    out[3] = mat[3] * x + mat[7] * y + mat[11] * z + mat[15] * w;
  
    if (out[3] !== 0) {
      out[0] /= out[3];
      out[1] /= out[3];
      out[2] /= out[3];
      out[3] /= out[3];
    }
  
    return out;
  }

Answer 1

Since the screen coordinates are 2D, while the world coordinates are 3D, the problem arises that you still have one degree of freedom open. A pixel on the screen therefore represents a line in world coordinates. Each position on this line is mapped exactly to the pixel on the screen. For this reason, we need to define a plane where the line intersects. The intersection point then only needs to be calculated.

First we define the plane in world space. To do this, we need the plane normal and a point within the plane:

planeNormal = vec3(0, 0, 1);
planePoint = vec3(0, 0, 0);

To define the line, we can then simply use the same screen coordinates with two different z values.

vec2 screenCoordinate = vec2(someXValue, someYValue); // normalized screen coordinates [-1, 1]
vec4 linePoint0ScreenSpace = vec4(screenCoordinate.x, screenCoordinate.y, 0.0f, 1.0f); //Note: the z-value is set to 0
vec4 linePoint1ScreenSpace = vec4(screenCoordinate.x, screenCoordinate.y, 1.0f, 1.0f); //Note: the z-value is set to 1

we use 4D points because the view and the projection matrix are 4x4 matrices.

Now we convert both points into world space by multiplying the inverse matrix of viewProjection with the two 4D points.

mat4 vpInverse = inverse(viewProjectionMatrix);
vec4 linePoint0WorldSpace = vpInverse * linePoint0ScreenSpace;
vec4 linePoint1WorldSpace = vpInverse * linePoint1ScreenSpace;

The following step is only required if your projection is not linear, e.g. for perspective projections. Otherwise, this step is not required:

linePoint0WorldSpace /= linePoint0WorldSpace.w;
linePoint1WorldSpace /= linePoint1WorldSpace.w;

Okay, what do we have now?

We have two points in world space that are mapped to exactly the same pixel in screen space. These two points define a line in world space.

Using this line and the defined plane at the top of this answer, we can calculate the intersection point using the following method:

vec3 intersectionRayPlane(vec3 rayDirection, vec3 rayPoint, vec3 planeNormal, vec3 planePoint)
{
    return rayPoint - rayDirection * dot(rayPoint - planePoint, planeNormal) / dot(rayDirection, planeNormal);
}

So let's put our line in the right format and use the intersection method to calculate the 3D position in world space:

vec3 rayDirection = linePoint1WorldSpace.xyz - linePoint0WorldSpace.xyz;
vec3 rayPoint = linePoint1WorldSpace.xyz;

vec3 positionWorldSpace = intersectionRayPlane(rayDirection, rayPoint, planeNormal, planePoint);

The result is in positionWorldSpace.