0
$\begingroup$

I'm kind of lost here as I'm faced with a graphical bug the origin of which I don't even know. In an attempt to recreate a shadertoy I saw that had TXAA, I made a function TemporalAA that takes in 3 buffers: the current frame buffer, the last frame prevbuffer, and the frame to which we render the anti aliased image aabuffer.

The original implementation: https://www.shadertoy.com/view/4tcXD2

The issue is that there's this weird overlay pattern, and I don't notice any edge smoothing whatsoever. Also note the brightly colored bands that the spheres get when in this overlay.

Weird artifacts

It's completely written in C.

Note: a frame consists of an uint8 width*height*3 array. (times three for three color channels. [0] = red, [1] = green, [2] = blue). Please ignore the fact that everything is static inline.

// YUV-RGB conversion routine from Hyper3D
static inline vec3 encodePalYuv(color3 clr)
{
    vec3 rgb = (vec3){ // pow = gamma correction
        pow((float)clr.r/255, 2),
        pow((float)clr.g/255, 2),
        pow((float)clr.b/255, 2)
    };
    return (vec3){
        dot3(rgb, (vec3){0.299f, 0.587f, 0.114f}),
        dot3(rgb, (vec3){-0.14713f, -0.28886f, 0.436f}),
        dot3(rgb, (vec3){0.615f, -0.51499f, -0.10001f})
    };
}

static inline vec3 decodePalYuv(vec3 yuv)
{
    return (vec3){ // pow = gamma correction
        pow(dot3(yuv, (vec3){1.f, 0.f, 1.13983}), 0.5),
        pow(dot3(yuv, (vec3){1.f, -0.39465, -0.58060}), 0.5),
        pow(dot3(yuv, (vec3){1.f, 2.03211, 0.f}), 0.5)
    };
}

// component wise multiplication
static inline vec3 compwisemult3(vec3 v, vec3 u){
    return (vec3){
        v.x*u.x,
        v.y*u.y,
        v.z*u.z
    };
}

static inline void TemporalAA(uint8_t * buffer, uint8_t * prevbuffer, uint8_t * aabuffer, int width, int height) // out vec4 fragColor, in vec2 fragCoord )
{
    int pitch = width*3;

    for (int u = 1; u < width-1; ++u)
    {
        for (int v = 1; v < height-1; ++v) // we cant do the edge/corner pixels. That would be undefined behaviour
        {
            // index is the position in the buffers at which the current pixel resides
            // buffer[index], +1 and +2 for green & blue respectively.
            int index = (u<<2) + v*pitch; // leftshift 2 == multiply 4 but faster

            vec3 antialiased = c3tov3( (color3){prevbuffer[index], prevbuffer[index+1], prevbuffer[index+2]} );
            // vec4 lastColor = texture(iChannel1, uv);
            float mixRate = 0.5f;//fminf(1/width, 0.5);

            //vec3 in0 = texture(iChannel0, uv).xyz;
            color3 in[9];
            in[0] = ( (color3){buffer[index], buffer[index+1], buffer[index+2]} );
            vec3 in0 = c3tov3(in[0]);

            antialiased = lerp3(compwisemult3(antialiased, antialiased), compwisemult3(in0, in0), mixRate);
            antialiased.x = sqrt(antialiased.x);
            antialiased.y = sqrt(antialiased.y);
            antialiased.z = sqrt(antialiased.z);

            // PaulHK:
            // index = (nu + nv*pitch)*3 == (nu<<2) + nv*pitch
            int nu, nv; //vec2 off = 1.0 / iResolution.xy;
            nu = u+1; nv = v; // vec3 in1 = texture(iChannel0, uv + vec2(+off.x, 0.0)).xyz;
            in[1] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u-1; nv = v; // vec3 in2 = texture(iChannel0, uv + vec2(-off.x, 0.0)).xyz;
            in[2] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u; nv = v+1; // vec3 in3 = texture(iChannel0, uv + vec2(0.0, +off.y)).xyz;
            in[3] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u; nv = v-1; // vec3 in4 = texture(iChannel0, uv + vec2(0.0, -off.y)).xyz;
            in[4] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u+1; nv = v+1; // vec3 in5 = texture(iChannel0, uv + vec2(+off.x, +off.y)).xyz;
            in[5] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u-1; nv = v+1; // vec3 in6 = texture(iChannel0, uv + vec2(-off.x, +off.y)).xyz;
            in[6] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u+1; nv = v-1; // vec3 in7 = texture(iChannel0, uv + vec2(+off.x, -off.y)).xyz;
            in[7] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u-1; nv = v-1; // vec3 in8 = texture(iChannel0, uv + vec2(-off.x, -off.y)).xyz;
            in[8] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});

            antialiased = encodePalYuv(v3toc3(antialiased));
                 in0 = encodePalYuv(in[0]);
            vec3 in1 = encodePalYuv(in[1]);
            vec3 in2 = encodePalYuv(in[2]);
            vec3 in3 = encodePalYuv(in[3]);
            vec3 in4 = encodePalYuv(in[4]);
            vec3 in5 = encodePalYuv(in[5]);
            vec3 in6 = encodePalYuv(in[6]);
            vec3 in7 = encodePalYuv(in[7]);
            vec3 in8 = encodePalYuv(in[8]);

            vec3 minColor;
            minColor.x = fminf(fminf(fminf(in0.x, in1.x), fminf(in2.x, in3.x)), in4.x);
            minColor.y = fminf(fminf(fminf(in0.y, in1.y), fminf(in2.y, in3.y)), in4.y);
            minColor.z = fminf(fminf(fminf(in0.z, in1.z), fminf(in2.z, in3.z)), in4.z);

            vec3 maxColor;
            maxColor.x = fmaxf(fmaxf(fmaxf(in0.x, in1.x), fmaxf(in2.x, in3.x)), in4.x);
            maxColor.y = fmaxf(fmaxf(fmaxf(in0.y, in1.y), fmaxf(in2.y, in3.y)), in4.y);
            maxColor.z = fmaxf(fmaxf(fmaxf(in0.z, in1.z), fmaxf(in2.z, in3.z)), in4.z);

            vec3 lerpto1;
            lerpto1.x = fminf(fminf(fminf(in5.x, in6.x), fminf(in7.x, in8.x)), minColor.x);
            lerpto1.y = fminf(fminf(fminf(in5.y, in6.y), fminf(in7.y, in8.y)), minColor.y);
            lerpto1.z = fminf(fminf(fminf(in5.z, in6.z), fminf(in7.z, in8.z)), minColor.z);

            minColor = lerp3(minColor, lerpto1, 0.5);

            vec3 lerpto2;
            lerpto2.x = fmaxf(fmaxf(fmaxf(in5.x, in6.x), fmaxf(in7.x, in8.x)), maxColor.x);
            lerpto2.y = fmaxf(fmaxf(fmaxf(in5.y, in6.y), fmaxf(in7.y, in8.y)), maxColor.y);
            lerpto2.z = fmaxf(fmaxf(fmaxf(in5.z, in6.z), fmaxf(in7.z, in8.z)), maxColor.z);

            maxColor = lerp3(maxColor, lerpto2, 0.5);

            vec3 preclamping = antialiased;
            antialiased.x = clampf(antialiased.x, minColor.x, maxColor.x);
            antialiased.y = clampf(antialiased.y, minColor.y, maxColor.y);
            antialiased.z = clampf(antialiased.z, minColor.z, maxColor.z);

            mixRate = 1.f / (1.f / mixRate + 1.f);

            vec3 diff = sub3(antialiased, preclamping);
            float clampAmount = dot3(diff, diff);

            mixRate += clampAmount * 4.f;
            mixRate = clampf(mixRate, 0.05f, 0.5f);

            antialiased = decodePalYuv(antialiased);

            //fragColor = vec4(antialiased, mixRate);
            color3 fragColor = v3toc3(scale3(antialiased, mixRate));
            aabuffer[index  ] = fragColor.r;
            aabuffer[index+1] = fragColor.g;
            aabuffer[index+2] = fragColor.b;
        }
    }
}
$\endgroup$
  • $\begingroup$ I'm curious about buffer[(nu<<2) + nv*pitch] >> I'm assuming you are using (nu, nv) to sample a 2d array of R,G,B bytes? if that's the case then the offset calculation should be (nu + nv * pitch) * sizeof(Pixel) - where Pixel is defined as { uint8char r,g,b } - I'm assuming pitch is in pixel units, not byte units. It may be better to encode your pixels as 4 bytes with a padding byte so you can keep your faster *4 offset calculation (you can also copy pixels via a single uint32_t type) $\endgroup$ – PaulHK Nov 6 at 2:09
  • $\begingroup$ @PaulHK the point was to get a 9x9 grid of the previous render centered on the current pixel. I figured that if I reused the equation for "index" but offset the vertical pixel or horizontal pixel by -1 or 1 or 0 I would get there. These aren't really uv coordinates in [0,1], they are coordinates in x [0, width] and y [0, height]. Since I haven't had trouble with the index equation in my main render loop I had no reason to suspect it was wrong. I will try your suggestion! $\endgroup$ – AnnoyinC Nov 6 at 7:56
  • $\begingroup$ Ahh, premature optimisation, the root of all evil :) $\endgroup$ – PaulHK Nov 6 at 8:02
  • $\begingroup$ While substituting your equation I made a mistake which should have made the output monocolor. This didn't happen, so now I know the 9x9 grid isn't being used correctly. $\endgroup$ – AnnoyinC Nov 6 at 8:07

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.