Rare bug in triangle clipping

[joestash]

Hi Vindar,

I noticed a rare corner case where large triangles (like a ground plane) will unexpectedly disappear when viewed up close at an angle. I think a vertex behind the camera is causing divide by negative w to trick _discardTriangle() into discarding, when a properly clipped region would intersect the screen.

A potential fix seems to be fully clipping (no early discard) when any w <= 0:

diff --git a/src/Renderer3D.h b/src/Renderer3D.h
index 3c136d4..bd1d18d 100644
--- a/src/Renderer3D.h
+++ b/src/Renderer3D.h
@@ -2938,6 +2938,9 @@ namespace tgx
          * coords are given after z-divide. */
         inline bool _discardTriangle(const fVec4 & P1, const fVec4 & P2, const fVec4 & P3)
             {
+            if ((P1.w <= 0) || (P2.w <= 0) || (P3.w <= 0))
+                return false; // do not discard
+
             const float bx = (_ox - 1) * _ilx - 1.0f;
             const float Bx = (_ox + _uni.im->width() + 1) * _ilx - 1.0f;
             const float by = (_oy - 1) * _ily - 1.0f;

Here is minimal example code that triggers the bad discard as the cube rotates:

#include <SDL.h>
#include <tgx.h>
using namespace tgx;

const int SCREEN_W = 480;
const int SCREEN_H = 320;
const int TEX_SIZE = 64;

uint16_t fb[SCREEN_W * SCREEN_H];
uint16_t zbuf[SCREEN_W * SCREEN_H];
uint16_t tbuf[TEX_SIZE * TEX_SIZE];

Image<RGB565> imfb(fb, SCREEN_W, SCREEN_H);
Image<RGB565> imtex(tbuf, TEX_SIZE, TEX_SIZE);

const Shader LOADED_SHADERS = SHADER_PERSPECTIVE | SHADER_ZBUFFER | SHADER_UNLIT |
                              SHADER_TEXTURE | SHADER_TEXTURE_NEAREST | SHADER_TEXTURE_WRAP_POW2;
Renderer3D<RGB565, LOADED_SHADERS, uint16_t> renderer;

int main(int argc, char *argv[]) {

    SDL_Init(SDL_INIT_VIDEO);
    SDL_Window *win = SDL_CreateWindow("BugTest", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, SCREEN_W, SCREEN_H, 0);
    SDL_Renderer *rnd = SDL_CreateRenderer(win, -1, SDL_RENDERER_PRESENTVSYNC | SDL_RENDERER_SOFTWARE);
    SDL_Texture *tex = SDL_CreateTexture(rnd, SDL_PIXELFORMAT_RGB565, SDL_TEXTUREACCESS_STREAMING, SCREEN_W, SCREEN_H);

    renderer.setViewportSize(SCREEN_W, SCREEN_H);
    renderer.setOffset(0, 0);
    renderer.setImage(&imfb);
    renderer.setZbuffer(zbuf);
    renderer.setCulling(0);
    renderer.setTextureQuality(SHADER_TEXTURE_NEAREST);
    renderer.setTextureWrappingMode(SHADER_TEXTURE_WRAP_POW2);
    renderer.setShaders(SHADER_UNLIT | SHADER_TEXTURE);
    renderer.setPerspective(45.0f, (float)SCREEN_W / SCREEN_H, 0.01f, 5.0f);

    renderer.setLookAt({ 1.0f, 1.0f, 1.0f },
                       { 0.0f, 0.5f, 0.0f },
                       { 0.0f, 1.0f, 0.0f });

    // checkered texture
    imtex.clear(RGB565_Black);
    for (int i = 0; i < TEX_SIZE; i += 4) {
        imtex.drawFastVLine({ i, 0 }, TEX_SIZE, RGB565_Teal);
        imtex.drawFastHLine({ 0, i }, TEX_SIZE, RGB565_Teal);
    }

    while (true) {

        for (SDL_Event e; SDL_PollEvent(&e);) {
            if (e.type == SDL_QUIT) return 0;
        }

        imfb.clear(RGB565_Black);
        renderer.clearZbuffer();

        float angle = fmodf((float)SDL_GetTicks() * 0.05f, 360.0f);
        renderer.setModelPosScaleRot({ 0.0f, 0.0f, 0.0f },
                                     { 5.0f, 0.5f, 5.0f },
                                     angle,
                                     { 0.0f, 1.0f, 0.0f });

        renderer.drawCube(&imtex, &imtex, &imtex, &imtex, &imtex, &imtex);

        SDL_UpdateTexture(tex, NULL, fb, SCREEN_W * sizeof(uint16_t));
        SDL_RenderClear(rnd);
        SDL_RenderCopy(rnd, tex, NULL, NULL);
        SDL_RenderPresent(rnd);
    }
}