I'm using GLSL 4.0 to write a subdivision routine that divides the triangles on an object through a geometry shader. I'm not using barycentric co-ordinates but vector arithmetic to output the triangle vertices. I managed to get the subdivision working but I don't understand how to interpolate the vertex normals to produce the correct shading:

#version 400 

layout(triangles) in;
layout(triangle_strip, max_vertices = 93) out; //MAX 128

uniform mat4 mvMatrix;
uniform mat4 pMatrix;
uniform mat3 normalMatrix; //mv matrix without translation
uniform int subdivision_level;
uniform vec4 lightPosition_camSpace; //light Position in camera space

uniform int time;

in data
  vec4 position_camSpace;
  vec3 normal_camSpace;
  vec2 textureCoordinate;
  vec4 color;

out fragmentData
  vec4 position_camSpace;
  vec3 normal_camSpace;
  vec2 textureCoordinate;
  vec4 color;
} frag;

void emit_triangle(vec4 position, vec4 width, vec4 hyp, float s) {


    gl_Position = position;
    frag.color = vertexIn[0].color;
    frag.position_camSpace = vertexIn[0].position_camSpace;
    frag.normal_camSpace = vertexIn[0].normal_camSpace;
    frag.textureCoordinate = vertexIn[0].textureCoordinate;

    gl_Position = position + width;
    frag.color = vertexIn[1].color;
    frag.position_camSpace = vertexIn[1].position_camSpace;
    frag.normal_camSpace = vertexIn[1].normal_camSpace;
    frag.textureCoordinate = vertexIn[1].textureCoordinate;


    gl_Position = position + hyp;
    frag.color = vertexIn[2].color;
    frag.position_camSpace = vertexIn[2].position_camSpace;
    frag.normal_camSpace = vertexIn[2].normal_camSpace;
    frag.textureCoordinate = vertexIn[2].textureCoordinate;



void main() {

  // A gl_In[0].gl_Position;
  // B gl_In[1].gl_Position;
  // C gl_In[2].gl_Position;

  float s = 1/pow(2,subdivision_level);
  int triangles_render = int(1/s);

  vec4 hyp_off = vec4(0.0,0.0,0.0,0.0);
  vec4 width_off = vec4(0.0,0.0,0.0,0.0);

  vec4 position = gl_in[0].gl_Position;
  vec4 width = s*(gl_in[1].gl_Position - gl_in[0].gl_Position);
  vec4 hyp = s*(gl_in[2].gl_Position - gl_in[0].gl_Position);

  vec4 hyp_off_f = vec4(0.0,0.0,0.0,0.0);
  vec4 width_off_f = width;

  for (int i=0; i<=subdivision_level+1; i++) {
    for (int j=0; j<triangles_render-i; j++) {

      // Normal triangle
      emit_triangle(position + hyp_off + width_off, width, hyp,s);
      width_off += width;  

      // Flipped triangle
      if (j <triangles_render-i-1)
        emit_triangle(position + width_off_f + hyp_off_f, hyp, hyp - width,s);
        width_off_f += width;

    hyp_off += hyp;
    width_off = vec4(0.0,0.0,0.0,0.0);

    hyp_off_f += hyp;
    width_off_f = width;


Currently the result looks like flat shading because I just use the normals of the vertices from the vertex shader. How do I compute the correct normals so that the shading looks correct? Also do I need to modify the vertex position too? I'm unsure about this part. I've read https://stackoverflow.com/questions/4703432/why-does-my-opengl-phong-shader-behave-like-a-flat-shader But they use barycentric co-ordinates for the solution.

  • 1
    $\begingroup$ max_vertices = 93) out; //MAX 128 What does that mean? You specified a max of 93, so why does your comment say otherwise? $\endgroup$ – Nicol Bolas Feb 14 '17 at 16:44
  • 3
    $\begingroup$ Also, it's not clear what this subdivision form is trying to do. It looks vaguely like you're linearly interpolating between the triangle positions (in clip space, no less). I'm not sure what that's supposed to accomplish or what the "correct" normals for that would even be. $\endgroup$ – Nicol Bolas Feb 14 '17 at 17:02

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.