git.y1.nz

commit 644e0fad3598708859acb52f563c44fa71bbefba
parent 42593c776ca9db096d8dfe61561d47204c7bd03e
Author: Toxa <56631470+untoxa@users.noreply.github.com>
Date:   Mon,  6 Feb 2023 00:18:40 +0400

Merge pull request #473 from michel-iwaniec/png2asset_palette_sort_bugfix

png2asset: Fix bug with incorrect palettes when different colors have same luma
Diffstat:
M
gbdk-support/png2asset/png2asset.cpp
21
++++++++++++++-------

1 file changed, 14 insertions(+), 7 deletions(-)
diff --git a/gbdk-support/png2asset/png2asset.cpp b/gbdk-support/png2asset/png2asset.cpp
@@ -425,9 +425,9 @@ void GetMap()
 }
 //Functor to compare entries in SetPal
 struct CmpIntColor {
-  bool operator() (unsigned int const& c1, unsigned int const& c2) const
-  {
-	unsigned char* c1_ptr = (unsigned char*)&c1;
+	bool operator() (unsigned int const& c1, unsigned int const& c2) const
+	{
+		unsigned char* c1_ptr = (unsigned char*)&c1;
 		unsigned char* c2_ptr = (unsigned char*)&c2;
 
 		//Compare alpha first, transparent color is considered smaller
@@ -437,11 +437,18 @@ struct CmpIntColor {
 		}
 		else
 		{
-			unsigned int lum_1 = (unsigned int)(c1_ptr[3] * 0.299f + c1_ptr[2] * 0.587f + c1_ptr[1] * 0.114f);
-			unsigned int lum_2 = (unsigned int)(c2_ptr[3] * 0.299f + c2_ptr[2] * 0.587f + c2_ptr[1] * 0.114f);
-			return lum_1 > lum_2;
+			// Do a compare with luminance in upper bits, and original rgb24 in lower bits. 
+			// This prefers luminance, but considers RGB values for equal-luminance cases to
+			// make sure the compare functor satisifed the strictly weak ordering requirement
+			uint64_t lum_1 = (unsigned int)(c1_ptr[3] * 299 + c1_ptr[2] * 587 + c1_ptr[1] * 114);
+			uint64_t lum_2 = (unsigned int)(c2_ptr[3] * 299 + c2_ptr[2] * 587 + c2_ptr[1] * 114);
+			uint64_t rgb_1 = (c1_ptr[3] << 16) | (c1_ptr[2] << 8) | c1_ptr[1];
+			uint64_t rgb_2 = (c2_ptr[3] << 16) | (c2_ptr[2] << 8) | c2_ptr[1];
+			uint64_t all_1 = (lum_1 << 24) | rgb_1;
+			uint64_t all_2 = (lum_2 << 24) | rgb_2;
+			return all_1 > all_2;
 		}
-  }
+	}
 };
 
 //This set will keep colors in the palette ordered based on their grayscale values to ensure they look good on DMG

This webpage is intended to be an accessible preview of this repository. To get a fuller picture, clone it and use the git CLI.