0
$\begingroup$

I'm kind of lost here as I'm faced with a graphical bug the origin of which I don't even know. In an attempt to recreate a shadertoy I saw that had TXAA, I made a function TemporalAA that takes in 3 buffers: the current frame buffer, the last frame prevbuffer, and the frame to which we render the anti aliased image aabuffer.

The original implementation: https://www.shadertoy.com/view/4tcXD2

The issue is that there's this weird overlay pattern, and I don't notice any edge smoothing whatsoever. Also note the brightly colored bands that the spheres get when in this overlay.

Weird artifacts

It's completely written in C.

Note: a frame consists of an uint8 width*height*3 array. (times three for three color channels. [0] = red, [1] = green, [2] = blue). Please ignore the fact that everything is static inline.

// YUV-RGB conversion routine from Hyper3D
static inline vec3 encodePalYuv(color3 clr)
{
    vec3 rgb = (vec3){ // pow = gamma correction
        pow((float)clr.r/255, 2),
        pow((float)clr.g/255, 2),
        pow((float)clr.b/255, 2)
    };
    return (vec3){
        dot3(rgb, (vec3){0.299f, 0.587f, 0.114f}),
        dot3(rgb, (vec3){-0.14713f, -0.28886f, 0.436f}),
        dot3(rgb, (vec3){0.615f, -0.51499f, -0.10001f})
    };
}

static inline vec3 decodePalYuv(vec3 yuv)
{
    return (vec3){ // pow = gamma correction
        pow(dot3(yuv, (vec3){1.f, 0.f, 1.13983}), 0.5),
        pow(dot3(yuv, (vec3){1.f, -0.39465, -0.58060}), 0.5),
        pow(dot3(yuv, (vec3){1.f, 2.03211, 0.f}), 0.5)
    };
}

// component wise multiplication
static inline vec3 compwisemult3(vec3 v, vec3 u){
    return (vec3){
        v.x*u.x,
        v.y*u.y,
        v.z*u.z
    };
}

static inline void TemporalAA(uint8_t * buffer, uint8_t * prevbuffer, uint8_t * aabuffer, int width, int height) // out vec4 fragColor, in vec2 fragCoord )
{
    int pitch = width*3;

    for (int u = 1; u < width-1; ++u)
    {
        for (int v = 1; v < height-1; ++v) // we cant do the edge/corner pixels. That would be undefined behaviour
        {
            // index is the position in the buffers at which the current pixel resides
            // buffer[index], +1 and +2 for green & blue respectively.
            int index = (u<<2) + v*pitch; // leftshift 2 == multiply 4 but faster

            vec3 antialiased = c3tov3( (color3){prevbuffer[index], prevbuffer[index+1], prevbuffer[index+2]} );
            // vec4 lastColor = texture(iChannel1, uv);
            float mixRate = 0.5f;//fminf(1/width, 0.5);

            //vec3 in0 = texture(iChannel0, uv).xyz;
            color3 in[9];
            in[0] = ( (color3){buffer[index], buffer[index+1], buffer[index+2]} );
            vec3 in0 = c3tov3(in[0]);

            antialiased = lerp3(compwisemult3(antialiased, antialiased), compwisemult3(in0, in0), mixRate);
            antialiased.x = sqrt(antialiased.x);
            antialiased.y = sqrt(antialiased.y);
            antialiased.z = sqrt(antialiased.z);

            // PaulHK:
            // index = (nu + nv*pitch)*3 == (nu<<2) + nv*pitch
            int nu, nv; //vec2 off = 1.0 / iResolution.xy;
            nu = u+1; nv = v; // vec3 in1 = texture(iChannel0, uv + vec2(+off.x, 0.0)).xyz;
            in[1] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u-1; nv = v; // vec3 in2 = texture(iChannel0, uv + vec2(-off.x, 0.0)).xyz;
            in[2] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u; nv = v+1; // vec3 in3 = texture(iChannel0, uv + vec2(0.0, +off.y)).xyz;
            in[3] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u; nv = v-1; // vec3 in4 = texture(iChannel0, uv + vec2(0.0, -off.y)).xyz;
            in[4] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u+1; nv = v+1; // vec3 in5 = texture(iChannel0, uv + vec2(+off.x, +off.y)).xyz;
            in[5] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u-1; nv = v+1; // vec3 in6 = texture(iChannel0, uv + vec2(-off.x, +off.y)).xyz;
            in[6] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u+1; nv = v-1; // vec3 in7 = texture(iChannel0, uv + vec2(+off.x, -off.y)).xyz;
            in[7] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});
            nu = u-1; nv = v-1; // vec3 in8 = texture(iChannel0, uv + vec2(-off.x, -off.y)).xyz;
            in[8] = ((color3){buffer[(nu + nv*pitch)*3], buffer[(nu + nv*pitch)*3 +1], buffer[(nu + nv*pitch)*3 +2]});

            antialiased = encodePalYuv(v3toc3(antialiased));
                 in0 = encodePalYuv(in[0]);
            vec3 in1 = encodePalYuv(in[1]);
            vec3 in2 = encodePalYuv(in[2]);
            vec3 in3 = encodePalYuv(in[3]);
            vec3 in4 = encodePalYuv(in[4]);
            vec3 in5 = encodePalYuv(in[5]);
            vec3 in6 = encodePalYuv(in[6]);
            vec3 in7 = encodePalYuv(in[7]);
            vec3 in8 = encodePalYuv(in[8]);

            vec3 minColor;
            minColor.x = fminf(fminf(fminf(in0.x, in1.x), fminf(in2.x, in3.x)), in4.x);
            minColor.y = fminf(fminf(fminf(in0.y, in1.y), fminf(in2.y, in3.y)), in4.y);
            minColor.z = fminf(fminf(fminf(in0.z, in1.z), fminf(in2.z, in3.z)), in4.z);

            vec3 maxColor;
            maxColor.x = fmaxf(fmaxf(fmaxf(in0.x, in1.x), fmaxf(in2.x, in3.x)), in4.x);
            maxColor.y = fmaxf(fmaxf(fmaxf(in0.y, in1.y), fmaxf(in2.y, in3.y)), in4.y);
            maxColor.z = fmaxf(fmaxf(fmaxf(in0.z, in1.z), fmaxf(in2.z, in3.z)), in4.z);

            vec3 lerpto1;
            lerpto1.x = fminf(fminf(fminf(in5.x, in6.x), fminf(in7.x, in8.x)), minColor.x);
            lerpto1.y = fminf(fminf(fminf(in5.y, in6.y), fminf(in7.y, in8.y)), minColor.y);
            lerpto1.z = fminf(fminf(fminf(in5.z, in6.z), fminf(in7.z, in8.z)), minColor.z);

            minColor = lerp3(minColor, lerpto1, 0.5);

            vec3 lerpto2;
            lerpto2.x = fmaxf(fmaxf(fmaxf(in5.x, in6.x), fmaxf(in7.x, in8.x)), maxColor.x);
            lerpto2.y = fmaxf(fmaxf(fmaxf(in5.y, in6.y), fmaxf(in7.y, in8.y)), maxColor.y);
            lerpto2.z = fmaxf(fmaxf(fmaxf(in5.z, in6.z), fmaxf(in7.z, in8.z)), maxColor.z);

            maxColor = lerp3(maxColor, lerpto2, 0.5);

            vec3 preclamping = antialiased;
            antialiased.x = clampf(antialiased.x, minColor.x, maxColor.x);
            antialiased.y = clampf(antialiased.y, minColor.y, maxColor.y);
            antialiased.z = clampf(antialiased.z, minColor.z, maxColor.z);

            mixRate = 1.f / (1.f / mixRate + 1.f);

            vec3 diff = sub3(antialiased, preclamping);
            float clampAmount = dot3(diff, diff);

            mixRate += clampAmount * 4.f;
            mixRate = clampf(mixRate, 0.05f, 0.5f);

            antialiased = decodePalYuv(antialiased);

            //fragColor = vec4(antialiased, mixRate);
            color3 fragColor = v3toc3(scale3(antialiased, mixRate));
            aabuffer[index  ] = fragColor.r;
            aabuffer[index+1] = fragColor.g;
            aabuffer[index+2] = fragColor.b;
        }
    }
}
$\endgroup$
4
  • $\begingroup$ I'm curious about buffer[(nu<<2) + nv*pitch] >> I'm assuming you are using (nu, nv) to sample a 2d array of R,G,B bytes? if that's the case then the offset calculation should be (nu + nv * pitch) * sizeof(Pixel) - where Pixel is defined as { uint8char r,g,b } - I'm assuming pitch is in pixel units, not byte units. It may be better to encode your pixels as 4 bytes with a padding byte so you can keep your faster *4 offset calculation (you can also copy pixels via a single uint32_t type) $\endgroup$
    – PaulHK
    Nov 6, 2019 at 2:09
  • $\begingroup$ @PaulHK the point was to get a 9x9 grid of the previous render centered on the current pixel. I figured that if I reused the equation for "index" but offset the vertical pixel or horizontal pixel by -1 or 1 or 0 I would get there. These aren't really uv coordinates in [0,1], they are coordinates in x [0, width] and y [0, height]. Since I haven't had trouble with the index equation in my main render loop I had no reason to suspect it was wrong. I will try your suggestion! $\endgroup$
    – AnnoyinC
    Nov 6, 2019 at 7:56
  • $\begingroup$ Ahh, premature optimisation, the root of all evil :) $\endgroup$
    – PaulHK
    Nov 6, 2019 at 8:02
  • $\begingroup$ While substituting your equation I made a mistake which should have made the output monocolor. This didn't happen, so now I know the 9x9 grid isn't being used correctly. $\endgroup$
    – AnnoyinC
    Nov 6, 2019 at 8:07

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.