I would like to display arbitrary 3d meshes with black edges (black outline, black ridges, etc.). Thereby I have created two different textures: One color texture and one anti-aliased edge texture.

Here is an example of the color texture:

enter image description here

And the the corresponding edge texture:

enter image description here

Now I'd like to merge both textures. The edge texture has a white background with black/grey edges. I used fxaa, therefore some pixels are grey.

My question: How can I combine (blend) both textures correctly? And do I have to apply fxaa for the color texture, too?

EDIT: For completeness, here's the fxaa-shader code:

    #version 330 core

in vec2 texPos;
out vec4 color;
uniform sampler2D colorTexture;
uniform float rt_w;
uniform float rt_h;

vec4 PostFX(sampler2D tex);
vec3 FxaaPixelShader(sampler2D tex, vec2 rcpFrame);  

void main()
    color = PostFX(colorTexture);
    //color = texelFetch(colorTexture, ivec2(gl_FragCoord), 0 );

vec4 PostFX(sampler2D tex)
  vec4 c = vec4(0.0);
  vec2 rcpFrame = vec2(1.0/rt_w, 1.0/rt_h);
  c.rgb = FxaaPixelShader(tex, rcpFrame);
  c.a = 1.0;
  return c;

vec3 FxaaPixelShader( 
  sampler2D tex, // Input texture.
  vec2 rcpFrame) // Constant {1.0/frameWidth, 1.0/frameHeight}.
    #define FXAA_REDUCE_MIN   (1.0/128.0)
    #define FXAA_REDUCE_MUL   (1.0/8.0)
    #define FXAA_SPAN_MAX     8.0
    vec3 rgbNW = texture(tex, vec2(texPos) + (vec2(-1,-1) * rcpFrame.xy)).xyz;
    vec3 rgbNE = texture(tex, vec2(texPos) + (vec2(1,-1) * rcpFrame.xy)).xyz;
    vec3 rgbSW = texture(tex, vec2(texPos)+ (vec2(-1,1) * rcpFrame.xy)).xyz;
    vec3 rgbSE = texture(tex, vec2(texPos) + (vec2(1,1) * rcpFrame.xy)).xyz;
    vec3 rgbM  = texture(tex, vec2(texPos)).xyz;
    vec3 luma = vec3(0.299, 0.587, 0.114);
    float lumaNW = dot(rgbNW, luma);
    float lumaNE = dot(rgbNE, luma);
    float lumaSW = dot(rgbSW, luma);
    float lumaSE = dot(rgbSE, luma);
    float lumaM  = dot(rgbM,  luma);
    float lumaMin = min(lumaM, min(min(lumaNW, lumaNE), min(lumaSW, lumaSE)));
    float lumaMax = max(lumaM, max(max(lumaNW, lumaNE), max(lumaSW, lumaSE)));
    vec2 dir; 
    dir.x = -((lumaNW + lumaNE) - (lumaSW + lumaSE));
    dir.y =  ((lumaNW + lumaSW) - (lumaNE + lumaSE));
    float dirReduce = max(
        (lumaNW + lumaNE + lumaSW + lumaSE) * (0.25 * FXAA_REDUCE_MUL),
    float rcpDirMin = 1.0/(min(abs(dir.x), abs(dir.y)) + dirReduce);
    dir = min(vec2( FXAA_SPAN_MAX,  FXAA_SPAN_MAX), 
          max(vec2(-FXAA_SPAN_MAX, -FXAA_SPAN_MAX), 
          dir * rcpDirMin)) * rcpFrame.xy;
    vec3 rgbA = (1.0/2.0) * (
        texture(tex, texPos.xy + dir * (1.0/3.0 - 0.5)).xyz +
        texture(tex, texPos.xy + dir * (2.0/3.0 - 0.5)).xyz);
    vec3 rgbB = rgbA * (1.0/2.0) + (1.0/4.0) * (
        texture(tex, texPos.xy + dir * (0.0/3.0 - 0.5)).xyz +
        texture(tex, texPos.xy + dir * (3.0/3.0 - 0.5)).xyz);
    float lumaB = dot(rgbB, luma);
    if((lumaB < lumaMin) || (lumaB > lumaMax)) return rgbA;
    return rgbB; 
  • $\begingroup$ Would it help to replace the edge texture with one that uses pure black and several levels of alpha, rather than using several levels of grey? $\endgroup$
    – trichoplax
    Aug 21 '16 at 23:11
  • $\begingroup$ @trichoplax: I'd say yes, since I could use the glsl mix funtion. But I'm not sure how I'd have to change my fxaa-shader to use alpha instead of grey values. I added the fxaa shader code abvoe. $\endgroup$
    – enne87
    Aug 21 '16 at 23:23
  • $\begingroup$ I haven't tested this so I'll leave it to someone else to post an answer if they are confident it works, but I'd imagine it working something like this: For each pixel in the texture, change the alpha value to the opposite of the grey value, and set the colour to black (0, 0, 0). Black pixels become fully opaque, white pixels become fully transparent, so the final colour can simply be set to black for all pixels, since the ones you want to appear white will be fully transparent preventing the black from showing up. $\endgroup$
    – trichoplax
    Aug 21 '16 at 23:28
  • 2
    $\begingroup$ Something like: gl_FragColor = mix(colourRGB, vec3(0), 1 - edgeRGB.r); Would blend in your edge outlines as black $\endgroup$
    – PaulHK
    Aug 22 '16 at 2:18
  • 2
    $\begingroup$ I would suggest something very similar to @PaulHK's snippet too: combinedColor = mix(sceneColor, edgeColor, edgeOpacity * (1. - edgeRGB.r));. $\endgroup$ Aug 22 '16 at 3:17

The simplest solution would be just a multiplication, which would give black edges:

combinedColor = sceneColor * edgeRGB.r;

For more control I would suggest something very similar to what @PaulHK proposed in the comments:

float opacity = maxEdgeOpacity * (1. - edgeRGB.r);
combinedColor = mix(sceneColor, edgeColor, opacity);

edgeColor is the desired edge color and maxEdgeOpacity is meant to be a parameter between 0 (no edges at all) and 1 (opaque edges), that you both choose to your liking.


Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.