Prog on fixing psp alpha textures

src/display/texture.c

@@ -26,6 +26,7 @@ void textureInit(
   memoryZero(texture, sizeof(texture_t));
   texture->width = width;
   texture->height = height;
+  texture->format = format;
 
   #if PSP
     assertTrue(
@@ -50,20 +51,83 @@ void textureInit(
         );
         break;
 
-      case TEXTURE_FORMAT_ALPHA:
-        glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_COMBINE);
-        // RGB = vertex color
-        glTexEnvi(GL_TEXTURE_ENV, GL_COMBINE_RGB, GL_REPLACE);
-        glTexEnvi(GL_TEXTURE_ENV, GL_SOURCE0_RGB, GL_PRIMARY_COLOR);
-        // A = texture alpha
-        glTexEnvi(GL_TEXTURE_ENV, GL_COMBINE_ALPHA, GL_REPLACE);
-        glTexEnvi(GL_TEXTURE_ENV, GL_SOURCE0_ALPHA, GL_TEXTURE);
+      case TEXTURE_FORMAT_ALPHA: {
+        assertNotNull(data.alpha.data, "Alpha texture data cannot be NULL");
+        #if PSP
+          // PSP kinda broken with alpha textures, so we convert it to paletted
+          // texture here.
+          uint8_t colorCount = 0;
+          uint8_t alphas[UINT8_MAX] = { 0 };
+          uint8_t colorIndex[width * height];
+          for(int32_t i = 0; i < width * height; i++) {
+            uint8_t alpha = data.alpha.data[i];
 
-        glTexImage2D(
-          GL_TEXTURE_2D, 0, format, width, height, 0,
-          format, GL_UNSIGNED_BYTE, (void*)data.alpha.data
-        );
+            // Already in the palette?
+            bool_t exists = false;
+            uint8_t j = 0;
+            for(j = 0; j < colorCount; j++) {
+              if(alphas[j] != alpha) continue;
+              exists = true;
+              break;
+            }
+
+            // if found use existing index, otherwise add to palette
+            if(exists) {
+              colorIndex[i] = j;
+              continue;
+            }
+            
+
+            // Should never happen given the format, but just in case:
+            assertTrue(
+              colorCount < UINT8_MAX,
+              "Too many unique alpha values for paletted texture"
+            );
+            colorIndex[i] = colorCount;
+            alphas[colorCount] = alpha;
+            colorCount++;
+          }
+
+          // Pad color count to power of two for PSP
+          uint8_t paddedColorCount = mathNextPowTwo(colorCount);
+          printf(
+            "Converted alpha texture to paletted format with %u unique alpha values (padded to %u)\n",
+            colorCount, paddedColorCount
+          );
+
+          // Create a palette with the unique alpha values
+          texture->palette = (color4b_t*)memoryAllocate(
+            sizeof(color4b_t) * paddedColorCount
+          );
+          memoryZero(texture->palette, sizeof(color4b_t) * paddedColorCount);
+          for(uint8_t i = 0; i < colorCount; i++) {
+            texture->palette[i].r = 0;
+            texture->palette[i].g = 0;
+            texture->palette[i].b = 0;
+            texture->palette[i].a = alphas[i];
+          }
+          
+          printf("Uploading paletted texture to GPU\n");
+          texture->format = TEXTURE_FORMAT_PALETTE;
+          glTexImage2D(
+            GL_TEXTURE_2D,
+            0, GL_COLOR_INDEX8_EXT,
+            width, height,
+            0, GL_COLOR_INDEX8_EXT,
+            GL_UNSIGNED_BYTE, (void*)colorIndex
+          );
+          glColorTableEXT(
+            GL_TEXTURE_2D, GL_RGBA, paddedColorCount, GL_RGBA,
+            GL_UNSIGNED_BYTE, (const void*)texture->palette
+          );
+        #else
+          glTexImage2D(
+            GL_TEXTURE_2D, 0, format, width, height, 0,
+            format, GL_UNSIGNED_BYTE, (void*)data.alpha.data
+          );
+        #endif
         break;
+      }
 
       case TEXTURE_FORMAT_PALETTE:
         assertNotNull(data.palette.data, "Palette texture data cannot be NULL");
@@ -155,7 +219,6 @@ void textureInit(
     texture->ready = true;
   
   #elif DOLPHIN
-    texture->format = format;
 
     switch(format) {
       case TEXTURE_FORMAT_RGBA:
@@ -330,6 +393,12 @@ void textureDispose(texture_t *texture) {
 
   #if DISPLAY_SDL2
     glDeleteTextures(1, &texture->id);
+    #if PSP
+      if(texture->palette) {
+        memoryFree(texture->palette);
+        texture->palette = NULL;
+      }
+    #endif
   #elif DOLPHIN
     switch(texture->format) {
       case TEXTURE_FORMAT_RGBA:

src/display/texture.h

@@ -24,15 +24,18 @@ typedef enum {
 typedef struct {
   #if DISPLAY_SDL2
     GLuint id;
+    #if PSP
+      color4b_t *palette;
+    #endif
   #elif DOLPHIN
     GXTexObj texObj;
-    textureformat_t format;
     union {
       u16 *rgba;
       u8 *alpha;
     };
   #endif
-
+    
+  textureformat_t format;
   bool_t ready;
   int32_t width;
   int32_t height;