Fix depth buffer issue causing translucent sprites to not render properly in Polymost.

Use polymost_spriteHasTranslucency() and polymost_maskWallHasTranslucency() to determine if a sprite has translucency.
Optimize sprite rendering when in Polymost by only sorting sprites with translucency.

git-svn-id: https://svn.eduke32.com/eduke32@7612 1a8010ca-5511-0410-912e-c29ae57300e0

# Conflicts:
#	source/build/src/polymost.cpp

source/build/include/polymost.h

@@ -48,6 +48,9 @@ void polymost_fillpolygon(int32_t npoints);
 void polymost_initosdfuncs(void);
 void polymost_drawrooms(void);
 
+int32_t polymost_maskWallHasTranslucency(uwalltype const * const wall);
+int32_t polymost_spriteHasTranslucency(uspritetype const * const tspr);
+
 void polymost_resetVertexPointers(void);
 void polymost_disableProgram(void);
 void polymost_resetProgram(void);
@@ -233,6 +236,7 @@ enum pthtyp_flags {
     PTH_NOTRANSFIX = 256, // fixtransparency() bypassed
 
     PTH_INDEXED = 512,
+    PTH_ONEBITALPHA = 1024,
 };
 
 typedef struct pthtyp_t

source/build/src/engine.cpp

@@ -8418,12 +8418,34 @@ void renderDrawMasks(void)
 #else
 # define debugmask_add(dispidx, idx) do {} while (0)
 #endif
-    int32_t i;
+    int32_t i = spritesortcnt-1;
+    int32_t numSprites = spritesortcnt;
 
-    for (i=spritesortcnt-1; i>=0; i--)
+    if (videoGetRenderMode() == REND_POLYMOST)
+    {
+        spritesortcnt = 0;
+        int32_t back = i;
+        for (; i >= 0; --i)
+        {
+            if (polymost_spriteHasTranslucency(&tsprite[i]))
+            {
+                tspriteptr[spritesortcnt] = &tsprite[i];
+                ++spritesortcnt;
+            } else
+            {
+                tspriteptr[back] = &tsprite[i];
+                --back;
+            }
+        }
+    } else
+    {
+        for (; i >= 0; --i)
+        {
             tspriteptr[i] = &tsprite[i];
+        }
+    }
 
-    for (i=spritesortcnt-1; i>=0; i--)
+    for (i=spritesortcnt-1; i>=0; --i)
     {
         const int32_t xs = tspriteptr[i]->x-globalposx, ys = tspriteptr[i]->y-globalposy;
         const int32_t yp = dmulscale6(xs,cosviewingrangeglobalang,ys,sinviewingrangeglobalang);
@@ -8447,8 +8469,10 @@ killsprite:
             if (!modelp)
 #endif
             {
-                spritesortcnt--;  //Delete face sprite if on wrong side!
+                //Delete face sprite if on wrong side!
-                if (i != spritesortcnt)
+                --numSprites;
+                --spritesortcnt;
+                if (i != numSprites)
                 {
                     tspriteptr[i] = tspriteptr[spritesortcnt];
                     spritesxyz[i].x = spritesxyz[spritesortcnt].x;
@@ -8473,13 +8497,11 @@ killsprite:
                 swaplong(&spritesxyz[l].y,&spritesxyz[l+gap].y);
             }
 
-    if (spritesortcnt > 0)
-        spritesxyz[spritesortcnt].y = (spritesxyz[spritesortcnt-1].y^1);
-
     ys = spritesxyz[0].y; i = 0;
     for (bssize_t j=1; j<=spritesortcnt; j++)
     {
-        if (spritesxyz[j].y == ys)
+        if (j == spritesortcnt ||
+            spritesxyz[j].y == ys)
             continue;
 
         ys = spritesxyz[j].y;
@@ -8532,6 +8554,36 @@ killsprite:
     }
 
     videoBeginDrawing(); //{{{
+
+#ifdef USE_OPENGL
+    if (videoGetRenderMode() == REND_POLYMOST)
+    {
+        for (i = spritesortcnt; i < numSprites; ++i)
+        {
+            if (tspriteptr[i] != NULL)
+            {
+                debugmask_add(i | 32768, tspriteptr[i]->owner);
+                renderDrawSprite(i);
+
+                tspriteptr[i] = NULL;
+            }
+        }
+
+        for (i = 0; i < maskwallcnt;)
+        {
+            if (polymost_maskWallHasTranslucency((uwalltype *) &wall[thewall[maskwallcnt-1]]))
+            {
+                int16_t maskSwap = maskwall[i];
+                maskwall[i] = maskwall[maskwallcnt-1];
+                maskwall[maskwallcnt-1] = maskSwap;
+                ++i;
+            }
+            else
+                renderDrawMaskedWall(--maskwallcnt);
+        }
+    }
+#endif
+
 #if 0
         for (i=spritesortcnt-1; i>=0; i--)
         {
@@ -8581,11 +8633,7 @@ killsprite:
         while (i)
         {
             i--;
-            if (tspriteptr[i] != NULL
+            if (tspriteptr[i] != NULL)
-#ifdef USE_OPENGL
-                && (!(tspriteptr[i]->cstat & 1024) || videoGetRenderMode() != REND_POLYMOST)
-#endif
-               )
             {
                 vec2f_t spr;
                 auto const tspr = tspriteptr[i];
@@ -8668,44 +8716,24 @@ killsprite:
         renderDrawMaskedWall(maskwallcnt);
     }
 
-    i = spritesortcnt;
-
-    while (i)
-    {
-        i--;
-        if (tspriteptr[i] != NULL
-#ifdef USE_OPENGL
-            && (!(tspriteptr[i]->cstat & 1024) || videoGetRenderMode() != REND_POLYMOST)
-#endif
-           )
-        {
-            debugmask_add(i | 32768, tspriteptr[i]->owner);
-            renderDrawSprite(i);
-
-            tspriteptr[i] = NULL;
-        }
-    }
-
 #ifdef USE_OPENGL
     if (videoGetRenderMode() == REND_POLYMOST)
-    {
         glDepthMask(GL_FALSE);
-
+#endif
     while (spritesortcnt)
     {
-            spritesortcnt--;
+        --spritesortcnt;
         if (tspriteptr[spritesortcnt] != NULL)
         {
-                Bassert(tspriteptr[spritesortcnt]->cstat & 1024);
+            debugmask_add(i | 32768, tspriteptr[i]->owner);
             renderDrawSprite(spritesortcnt);
             tspriteptr[spritesortcnt] = NULL;
         }
     }
-
+#ifdef USE_OPENGL
+    if (videoGetRenderMode() == REND_POLYMOST)
         glDepthMask(GL_TRUE);
-    }
 #endif
-    spritesortcnt = 0;
 
 #ifdef POLYMER
     if (videoGetRenderMode() == REND_POLYMER)

source/build/src/polymost.cpp

@@ -1531,7 +1531,7 @@ static void gloadtile_art_indexed(int32_t dapic, int32_t dameth, pthtyp *pth, in
     pth->palnum = 0;
     pth->shade = 0;
     pth->effects = 0;
-    pth->flags = TO_PTH_CLAMPED(dameth) | TO_PTH_NOTRANSFIX(dameth) | PTH_HASALPHA | (npoty*PTH_NPOTWALL) | PTH_INDEXED;
+    pth->flags = TO_PTH_CLAMPED(dameth) | TO_PTH_NOTRANSFIX(dameth) | (PTH_HASALPHA|PTH_ONEBITALPHA) | (npoty*PTH_NPOTWALL) | PTH_INDEXED;
     pth->hicr = NULL;
 }
 
@@ -1721,7 +1721,7 @@ void gloadtile_art(int32_t dapic, int32_t dapal, int32_t tintpalnum, int32_t das
     pth->palnum = dapal;
     pth->shade = dashade;
     pth->effects = 0;
-    pth->flags = TO_PTH_CLAMPED(dameth) | TO_PTH_NOTRANSFIX(dameth) | (hasalpha*PTH_HASALPHA) | (npoty*PTH_NPOTWALL);
+    pth->flags = TO_PTH_CLAMPED(dameth) | TO_PTH_NOTRANSFIX(dameth) | (hasalpha*(PTH_HASALPHA|PTH_ONEBITALPHA)) | (npoty*PTH_NPOTWALL);
     pth->hicr = NULL;
 
     if (hasfullbright && !fullbrightloadingpass)
@@ -1775,6 +1775,7 @@ int32_t gloadtile_hi(int32_t dapic,int32_t dapalnum, int32_t facen, hicreplctyp
 
     int32_t startticks = timerGetTicks(), willprint = 0;
 
+    char onebitalpha = 1;
     char hasalpha;
     vec2_t siz = { 0, 0 }, tsiz = { 0, 0 };
 
@@ -1886,7 +1887,6 @@ int32_t gloadtile_hi(int32_t dapic,int32_t dapalnum, int32_t facen, hicreplctyp
         int32_t b = (glinfo.bgra) ? tint.b : tint.r;
 
         char al = 255;
-        char onebitalpha = 1;
 
         for (bssize_t y = 0, j = 0; y < tsiz.y; ++y, j += siz.x)
         {
@@ -1945,6 +1945,7 @@ int32_t gloadtile_hi(int32_t dapic,int32_t dapalnum, int32_t facen, hicreplctyp
         }
 
         hasalpha = (al != 255);
+        onebitalpha &= hasalpha;
 
         if ((!(dameth & DAMETH_CLAMPED)) || facen) //Duplicate texture pixels (wrapping tricks for non power of 2 texture sizes)
         {
@@ -2005,6 +2006,7 @@ int32_t gloadtile_hi(int32_t dapic,int32_t dapalnum, int32_t facen, hicreplctyp
     pth->effects = effect;
     pth->flags = TO_PTH_CLAMPED(dameth) | TO_PTH_NOTRANSFIX(dameth) |
                  PTH_HIGHTILE | ((facen>0) * PTH_SKYBOX) |
+                 (onebitalpha ? PTH_ONEBITALPHA : 0) |
                  (hasalpha ? PTH_HASALPHA : 0) |
                  ((hicr->flags & HICR_FORCEFILTER) ? PTH_FORCEFILTER : 0);
     pth->skyface = facen;
@@ -2053,6 +2055,49 @@ static inline pthtyp *our_texcache_fetch(int32_t dameth)
     return texcache_fetch(globalpicnum, globalpal, getpalookup(!(globalflags & GLOBAL_NO_GL_TILESHADES) ? globvis>>3 : 0, globalshade), dameth);
 }
 
+int32_t polymost_maskWallHasTranslucency(uwalltype const * const wall)
+{
+    if (wall->cstat & CSTAT_WALL_TRANSLUCENT)
+        return true;
+
+    //POGO: only hightiles may have translucency in their texture
+    if (!usehightile)
+        return false;
+
+    int method = DAMETH_MASK | DAMETH_WALL;
+    if (wall->cstat & CSTAT_WALL_TRANSLUCENT)
+        method = DAMETH_WALL | (((wall->cstat & CSTAT_WALL_TRANS_FLIP)) ? DAMETH_TRANS2 : DAMETH_TRANS1);
+
+    uint8_t pal = wall->pal;
+    if (palookup[pal] == NULL)
+        pal = 0;
+
+    pthtyp* pth = texcache_fetch(wall->picnum, pal, 0, method);
+    return pth && (pth->flags & PTH_HASALPHA) && !(pth->flags & PTH_ONEBITALPHA);
+}
+
+int32_t polymost_spriteHasTranslucency(uspritetype const * const tspr)
+{
+    if ((tspr->cstat & (CSTAT_SPRITE_TRANSLUCENT | CSTAT_SPRITE_RESERVED1)) ||
+        spriteext[tspr->owner].alpha)
+        return true;
+
+    //POGO: only hightiles may have translucency in their texture
+    if (!usehightile)
+        return false;
+
+    int32_t method = DAMETH_MASK | DAMETH_CLAMPED;
+    if (tspr->cstat & CSTAT_SPRITE_TRANSLUCENT)
+        method = DAMETH_CLAMPED | ((tspr->cstat & CSTAT_SPRITE_TRANSLUCENT_INVERT) ? DAMETH_TRANS2 : DAMETH_TRANS1);
+
+    uint8_t pal = tspr->shade;
+    if (palookup[pal] == NULL)
+        pal = 0;
+
+    pthtyp* pth = texcache_fetch(tspr->picnum, pal, 0, method);
+    return pth && (pth->flags & PTH_HASALPHA) && !(pth->flags & PTH_ONEBITALPHA);
+}
+
 
 static void polymost_updatePalette()
 {