After a whole day of brain stress, I found a method to enumerate them:
//floats equal? with respect of epsilon
bool isNearlyTheSame(float value0, float value1)
{
return abs(value0 - value1) < 0.001;
}
uint getMinIndex(vec3 vector)
{
uint y = uint(vector.y <= vector.x + 0.001 && vector.y < vector.z);
uint z = uint(vector.z < vector.x && vector.z <= vector.y + 0.001)*2;
return y + z;
}
//result: x: 0 => inner triangles, 1 => inner most vertex, 2 => outer vertices
//result: y: lowest barycentric value is: 0 => x, 1 => y, 2 => z
uvec2 getBarycentricCase()
{
uvec2 baryCase = uvec2(0,0);
baryCase.y = getMinIndex(gl_TessCoord);
if(gl_TessCoord.x == 0 || gl_TessCoord.y == 0 || gl_TessCoord.z == 0)
baryCase.x = 2;
if(isNearlyTheSame(gl_TessCoord.x, gl_TessCoord.y) && isNearlyTheSame(gl_TessCoord.y, gl_TessCoord.z))
baryCase.x = 1;
return baryCase;
}
//returns a third of the count of the inner vertices with respect to the tessLevel
uint getNummTessPart(uint tessLevel)
{
float oddAdd = mod(tessLevel, 2) == 0 ? 0 : 1;
return tessLevel / 2.0 * (tessLevel / 2.0 - 1) + oddAdd;
}
//returns the current patch vertex id with respect to this algorithm
uint getPatchVertexID()
{
uint result;
uvec2 baryCase = getBarycentricCase();
uint outerTesslevel0 = uint(gl_TessLevelOuter[0]);
uint outerTesslevel1 = uint(gl_TessLevelOuter[1]);
uint outerTesslevel2 = uint(gl_TessLevelOuter[2]);
uint innerTesslevel = uint(gl_TessLevelInner[0]);
uint numTessPartInner = uint(getNummTessPart(innerTesslevel));
float baseValue = min(min(gl_TessCoord.x,gl_TessCoord.y), gl_TessCoord.z);
float leftValue = baryCase.y == 0 ? gl_TessCoord.z : baryCase.y == 1 ? gl_TessCoord.x : baryCase.y == 2 ? gl_TessCoord.y : 0;
if(baryCase.x == 0) //inner vertices
{
float stepUp = 2.0 / 3.0 / innerTesslevel;
float stepLeft = 1.0 / innerTesslevel;
uint positionUp = uint(round(baseValue / stepUp));
uint positionLeft = uint(round(leftValue / stepLeft + (positionUp) * stepUp * innerTesslevel / 2 - positionUp));
result = numTessPartInner - uint(getNummTessPart(innerTesslevel - ((positionUp-1) * 2)));
result += positionLeft;
result += baryCase.y * numTessPartInner;
}
else if(baryCase.x == 1) //center vertex
{
result = numTessPartInner * 3;
}
else if(baryCase.x == 2) //outer
{
uint maxOuterTessLevel = max(max(outerTesslevel0, outerTesslevel1), outerTesslevel2);
bool hasCenter = (mod(innerTesslevel,2) == 0 && innerTesslevel > 1) || (maxOuterTessLevel > 1 && innerTesslevel < 1);
bool hasInner = innerTesslevel > 2;
uint outerAdd[3] = {0, outerTesslevel0, outerTesslevel0 + outerTesslevel1};
result = uint(hasInner) * numTessPartInner * 3;
result += uint(hasCenter);
result += uint(round(leftValue * gl_TessLevelOuter[baryCase.y])) + outerAdd[baryCase.y];
}
return result;
}
As can be seen in Figure 1, the algorithm first enumerates the inner vertices, dividing them into 3 parts. After the inner vertices, the innermost vertex (in case of an even innerTessLevel) is enumerated. At the end, the outer vertices are enumerated.

Figure 1: Enumeration sceme for: gl_TessLevelInner[0] = 10
, gl_TessLevelOuter[0] = 10
, gl_TessLevelOuter[1] = 10
, gl_TessLevelOuter[2] = 10
.
This algorithm works for all types of triangles tessellation, where innerTessLevel and outerTessLevels are integer values. The enumeration only works for equal_spacing
. As you can see from the code, the enumerated value is calculated by the following built-in variables of TES:
in vec3 gl_TessCoord
patch in float gl_TessLevelOuter[3]
patch in float gl_TessLevelInner[1]
I will try to improve the algorithm from time to time.... at the moment it is very confusing and full of branches... maybe we can avoid branching in the end.
Also important, when using this algorithm:
Multiple generated vertices within TES stage are on the same position. So the output of this algorithm gives you an enumeration with respect to the position. But not with respect to the vertex id. So multiple generated vertices will receive the same enumarated number at the end.
Please feel free to give me tips on how to improve the algorithm =)