Merge pull request #14683 from rouault/fix_14682

gdal raster rgb-to-palette: fix crash with transparency / very low number of colors

Author: rouault

alg/gdalmediancut.cpp

@@ -600,6 +600,7 @@ int GDALComputeMedianCutPCTInternal(
     /* ==================================================================== */
     /*      STEP 4: assign colors to all boxes                              */
     /* ==================================================================== */
+    if (nColorCounter > 0)
     {
         Colorbox *ptr = usedboxes;
         if (panPixelCountPerColorTableEntry)

apps/gdalalg_raster_rgb_to_palette.cpp

@@ -253,15 +253,18 @@ bool GDALRasterRGBToPaletteAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
                 oCT = std::move(*(poCT.get()));
             }
         }
-
-        m_colorCount = oCT.GetColorEntryCount();
     }
 
+    m_colorCount = oCT.GetColorEntryCount();
+
     if (m_dstNoData >= 0)
     {
         for (int i = std::min(255, m_colorCount); i > m_dstNoData; --i)
         {
-            oCT.SetColorEntry(i, oCT.GetColorEntry(i - 1));
+            // Create a temporary copy to avoid use after free when SetColorEntry()
+            // resizes the underlying vector.
+            const GDALColorEntry sEntry = *(oCT.GetColorEntry(i - 1));
+            oCT.SetColorEntry(i, &sEntry);
         }
 
         poTmpDS->GetRasterBand(1)->SetNoDataValue(m_dstNoData);

autotest/utilities/test_gdalalg_raster_rgb_to_palette.py

@@ -329,3 +329,35 @@ def test_gdalalg_raster_rgb_to_palette_bit_depth_8(tmp_vsimem):
     ) as alg:
         ds = alg.Output()
         assert ds.GetRasterBand(1).Checksum() == 7593
+
+
+def test_gdalalg_raster_rgb_to_palette_rgba_single_color():
+
+    src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1, 4)
+    src_ds.GetRasterBand(1).SetColorInterpretation(gdal.GCI_RedBand)
+    src_ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_GreenBand)
+    src_ds.GetRasterBand(3).SetColorInterpretation(gdal.GCI_BlueBand)
+    src_ds.GetRasterBand(4).SetColorInterpretation(gdal.GCI_AlphaBand)
+    src_ds.GetRasterBand(4).Fill(255)
+
+    with gdal.Run(get_alg(), input=src_ds, output="", output_format="MEM") as alg:
+        ds = alg.Output()
+        assert ds.GetRasterBand(1).GetColorTable().GetCount() == 2
+        assert ds.GetRasterBand(1).GetColorTable().GetColorEntry(0) == (0, 0, 0, 0)
+        assert ds.GetRasterBand(1).GetColorTable().GetColorEntry(1) == (0, 0, 0, 255)
+        assert ds.GetRasterBand(1).Checksum() == 1
+
+
+def test_gdalalg_raster_rgb_to_palette_rgba_all_transparent():
+
+    src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1, 4)
+    src_ds.GetRasterBand(1).SetColorInterpretation(gdal.GCI_RedBand)
+    src_ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_GreenBand)
+    src_ds.GetRasterBand(3).SetColorInterpretation(gdal.GCI_BlueBand)
+    src_ds.GetRasterBand(4).SetColorInterpretation(gdal.GCI_AlphaBand)
+
+    with gdal.Run(get_alg(), input=src_ds, output="", output_format="MEM") as alg:
+        ds = alg.Output()
+        assert ds.GetRasterBand(1).GetColorTable().GetCount() == 1
+        assert ds.GetRasterBand(1).GetColorTable().GetColorEntry(0) == (0, 0, 0, 0)
+        assert ds.GetRasterBand(1).Checksum() == 0