overview: warn in case of conflicting mask sources

Fixes OSGeo#13703

```
$ gdal raster overview add --external sample2.tif -r average
Warning 1: Raster band 1 of dataset sample2.tif has several conflicting mask sources:
- nodata value
- related to a raster band that is an alpha band
Only the nodata value will be taken into account.
0...10...20...30...40...50...60...70...80...90...100 - done.
```

Author: rouault

apps/gdalalg_raster_reproject.cpp

@@ -207,6 +207,7 @@ bool GDALRasterReprojectAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
 
     CPLStringList aosOptions;
     std::string outputFilename;
+
     if (ctxt.m_poNextUsableStep)
     {
         CPLAssert(CanHandleNextStep(ctxt.m_poNextUsableStep));

autotest/cpp/test_gdal.cpp

@@ -6477,4 +6477,72 @@ TEST_F(test_gdal, GDALGeoTransform)
     }
 }
 
+TEST_F(test_gdal, GDALRasterBand_HasConflictingMaskSources)
+{
+    auto poMemDrv = GetGDALDriverManager()->GetDriverByName("MEM");
+    if (!poMemDrv)
+    {
+        GTEST_SKIP() << "MEM driver missing";
+    }
+    else
+    {
+        {
+            auto poDS = std::unique_ptr<GDALDataset>(
+                poMemDrv->Create("", 1, 1, 1, GDT_Byte, nullptr));
+            poDS->GetRasterBand(1)->SetNoDataValue(1);
+
+            EXPECT_FALSE(poDS->GetRasterBand(1)->HasConflictingMaskSources());
+        }
+
+        {
+            auto poDS = std::unique_ptr<GDALDataset>(
+                poMemDrv->Create("", 1, 1, 1, GDT_Byte, nullptr));
+            poDS->CreateMaskBand(GMF_PER_DATASET);
+
+            EXPECT_FALSE(poDS->GetRasterBand(1)->HasConflictingMaskSources());
+        }
+
+        {
+            auto poDS = std::unique_ptr<GDALDataset>(
+                poMemDrv->Create("", 1, 1, 2, GDT_Byte, nullptr));
+            poDS->GetRasterBand(2)->SetColorInterpretation(GCI_AlphaBand);
+
+            EXPECT_FALSE(poDS->GetRasterBand(1)->HasConflictingMaskSources());
+        }
+
+        {
+            auto poDS = std::unique_ptr<GDALDataset>(
+                poMemDrv->Create("", 1, 1, 1, GDT_Byte, nullptr));
+            poDS->SetMetadataItem("NODATA_VALUES", "0");
+
+            EXPECT_FALSE(poDS->GetRasterBand(1)->HasConflictingMaskSources());
+        }
+
+        for (int i = 0; i < 4; ++i)
+        {
+            for (int j = i + 1; j < 4; ++j)
+            {
+                auto poDS = std::unique_ptr<GDALDataset>(
+                    poMemDrv->Create("", 1, 1, 2, GDT_Byte, nullptr));
+                if (i == 0)
+                    poDS->GetRasterBand(1)->SetNoDataValue(1);
+                if (i == 1 || j == 1)
+                    poDS->GetRasterBand(1)->CreateMaskBand(0);
+                if (i == 2 || j == 2)
+                    poDS->GetRasterBand(2)->SetColorInterpretation(
+                        GCI_AlphaBand);
+                if (i == 3 || j == 3)
+                    poDS->SetMetadataItem("NODATA_VALUES", "0");
+
+                EXPECT_TRUE(
+                    poDS->GetRasterBand(1)->HasConflictingMaskSources());
+                std::string osMsg;
+                EXPECT_TRUE(
+                    poDS->GetRasterBand(1)->HasConflictingMaskSources(&osMsg));
+                EXPECT_TRUE(!osMsg.empty());
+            }
+        }
+    }
+}
+
 }  // namespace

autotest/utilities/test_gdalalg_raster_overview.py

@@ -491,3 +491,19 @@ def test_gdalalg_overview_add_in_pipeline(tmp_vsimem):
         assert ds.GetRasterBand(1).GetOverview(0).Checksum() == 1192
 
     assert src_ds.GetRasterBand(1).GetOverviewCount() == 0
+
+
+###############################################################################
+
+
+def test_gdalalg_overview_add_warn_conflicting_mask_sources():
+
+    src_ds = gdal.GetDriverByName("MEM").Create("foo", 20, 20, 2)
+    src_ds.GetRasterBand(1).SetNoDataValue(255)
+    src_ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand)
+
+    with gdaltest.error_raised(
+        gdal.CE_Warning,
+        match="Raster band 1 of dataset foo has several conflicting mask sources:\n- nodata value\n- related to a raster band that is an alpha band\nOnly the nodata value will be taken into account",
+    ):
+        gdal.alg.raster.overview.add(input=src_ds, levels=[2, 4], resampling="average")

doc/source/api/gdalrasterband_cpp.rst

@@ -71,3 +71,9 @@ GDALRasterWindow class
 .. doxygenclass:: GDALRasterWindow
    :project: api
    :members:
+
+
+.. below is an allow-list for spelling checker.
+
+.. spelling:word-list::
+    posDetailMessage

frmts/gtiff/libtiff/tif_packbits.c

@@ -243,6 +243,8 @@ static int PackBitsDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s)
     long n;
     int b;
 
+    uint8_t *op0 = op;
+    tmsize_t occ0 = occ;
     (void)s;
     bp = (int8_t *)tif->tif_rawcp;
     cc = tif->tif_rawcc;

gcore/gdal_rasterband.h

@@ -122,6 +122,8 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject
         GSpacing nPixelSpace, GSpacing nLineSpace,
         GDALRasterIOExtraArg *psExtraArg) CPL_WARN_UNUSED_RESULT;
 
+    CPL_INTERNAL bool HasNoData() const;
+
   protected:
     GDALRasterBand();
     explicit GDALRasterBand(int bForceCachedIO);
@@ -552,6 +554,8 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject
     virtual CPLErr CreateMaskBand(int nFlagsIn);
     virtual bool IsMaskBand() const;
     virtual GDALMaskValueRange GetMaskValueRange() const;
+    bool HasConflictingMaskSources(std::string *posDetailMessage = nullptr,
+                                   bool bMentionPrioritarySource = true) const;
 
     virtual CPLVirtualMem *
     GetVirtualMemAuto(GDALRWFlag eRWFlag, int *pnPixelSpace,

gcore/gdalrasterband.cpp

@@ -9015,6 +9015,37 @@ CPLErr CPL_STDCALL GDALSetDefaultRAT(GDALRasterBandH hBand,
     return poBand->SetDefaultRAT(GDALRasterAttributeTable::FromHandle(hRAT));
 }
 
+/************************************************************************/
+/*                             HasNoData()                              */
+/************************************************************************/
+
+bool GDALRasterBand::HasNoData() const
+{
+    int bHaveNoDataRaw = FALSE;
+    bool bHaveNoData = false;
+    GDALRasterBand *poThis = const_cast<GDALRasterBand *>(this);
+    if (eDataType == GDT_Int64)
+    {
+        CPL_IGNORE_RET_VAL(poThis->GetNoDataValueAsInt64(&bHaveNoDataRaw));
+        bHaveNoData = CPL_TO_BOOL(bHaveNoDataRaw);
+    }
+    else if (eDataType == GDT_UInt64)
+    {
+        CPL_IGNORE_RET_VAL(poThis->GetNoDataValueAsUInt64(&bHaveNoDataRaw));
+        bHaveNoData = CPL_TO_BOOL(bHaveNoDataRaw);
+    }
+    else
+    {
+        const double dfNoDataValue = poThis->GetNoDataValue(&bHaveNoDataRaw);
+        if (bHaveNoDataRaw &&
+            GDALNoDataMaskBand::IsNoDataInRange(dfNoDataValue, eDataType))
+        {
+            bHaveNoData = true;
+        }
+    }
+    return bHaveNoData;
+}
+
 /************************************************************************/
 /*                            GetMaskBand()                             */
 /************************************************************************/
@@ -9072,32 +9103,6 @@ CPLErr CPL_STDCALL GDALSetDefaultRAT(GDALRasterBandH hBand,
 GDALRasterBand *GDALRasterBand::GetMaskBand()
 
 {
-    const auto HasNoData = [this]()
-    {
-        int bHaveNoDataRaw = FALSE;
-        bool bHaveNoData = false;
-        if (eDataType == GDT_Int64)
-        {
-            CPL_IGNORE_RET_VAL(GetNoDataValueAsInt64(&bHaveNoDataRaw));
-            bHaveNoData = CPL_TO_BOOL(bHaveNoDataRaw);
-        }
-        else if (eDataType == GDT_UInt64)
-        {
-            CPL_IGNORE_RET_VAL(GetNoDataValueAsUInt64(&bHaveNoDataRaw));
-            bHaveNoData = CPL_TO_BOOL(bHaveNoDataRaw);
-        }
-        else
-        {
-            const double dfNoDataValue = GetNoDataValue(&bHaveNoDataRaw);
-            if (bHaveNoDataRaw &&
-                GDALNoDataMaskBand::IsNoDataInRange(dfNoDataValue, eDataType))
-            {
-                bHaveNoData = true;
-            }
-        }
-        return bHaveNoData;
-    };
-
     if (poMask != nullptr)
     {
         if (poMask.IsOwned())
@@ -9599,6 +9604,78 @@ GDALMaskValueRange GDALRasterBand::GetMaskValueRange() const
     return GMVR_UNKNOWN;
 }
 
+/************************************************************************/
+/*                      HasConflictingMaskSources()                     */
+/************************************************************************/
+
+/**
+ * \brief Returns whether a raster band has conflicting mask sources.
+ *
+ * That is, if more than one of the following conditions is met:
+ * - it has a binary mask band (that is not an alpha band)
+ * - it has an external mask flags (.msk file)
+ * - it has a nodata value
+ * - it belongs to a dataset with the NODATA_VALUES metadata item set
+ * - it belongs to a dataset that has a band with a GCI_AlphaBand color interpretation
+ *
+ * @param[out] posDetailMessage Pointer to a string that will contain the
+ *                              details of the conflict.
+ * @param bMentionPrioritarySource Whether the mask source used should be
+ *                                 mentioned in *posDetailMessage.
+ * @since GDAL 3.13.0
+ */
+
+bool GDALRasterBand::HasConflictingMaskSources(
+    std::string *posDetailMessage, bool bMentionPrioritarySource) const
+{
+    const bool bHasExternalMask = poDS && poDS->oOvManager.HaveMaskFile();
+    const bool bHasBinaryMaskBand =
+        ((const_cast<GDALRasterBand *>(this)->GetMaskFlags() &
+          (GMF_ALL_VALID | GMF_NODATA | GMF_ALPHA)) == 0) &&
+        (!bHasExternalMask || poDS->oOvManager.GetMaskBand(nBand) != this);
+    const bool bHasNoData = HasNoData();
+    const bool bHasNODATA_VALUES =
+        poDS && poDS->GetMetadataItem("NODATA_VALUES");
+    const bool bHasAlphaBand =
+        poDS &&
+        poDS->GetRasterBand(poDS->GetRasterCount())->GetColorInterpretation() ==
+            GCI_AlphaBand;
+    const bool abMaskSources[] = {bHasBinaryMaskBand, bHasExternalMask,
+                                  bHasNoData, bHasNODATA_VALUES, bHasAlphaBand};
+    const size_t nCount =
+        std::count(std::begin(abMaskSources), std::end(abMaskSources), true);
+    if (nCount >= 2)
+    {
+        if (posDetailMessage)
+        {
+            *posDetailMessage = "Raster band ";
+            *posDetailMessage += std::to_string(nBand);
+            if (poDS && poDS->GetDescription()[0])
+            {
+                *posDetailMessage += " of dataset ";
+                *posDetailMessage += poDS->GetDescription();
+            }
+            *posDetailMessage += " has several conflicting mask sources:\n";
+            if (bHasExternalMask)
+                *posDetailMessage += "- internal binary mask band\n";
+            if (bHasExternalMask)
+                *posDetailMessage += "- external mask band (.msk)\n";
+            if (bHasNoData)
+                *posDetailMessage += "- nodata value\n";
+            if (bHasNODATA_VALUES)
+                *posDetailMessage += "- NODATA_VALUES dataset metadata item\n";
+            if (bHasAlphaBand)
+                *posDetailMessage +=
+                    "- related to a raster band that is an alpha band\n";
+            if (bMentionPrioritarySource)
+                *posDetailMessage +=
+                    "Only the first listed one will be taken into account.";
+        }
+        return true;
+    }
+    return false;
+}
+
 /************************************************************************/
 /*                    GetIndexColorTranslationTo()                      */
 /************************************************************************/

gcore/overview.cpp

@@ -4559,6 +4559,8 @@ static CPLErr GDALRegenerateCascadingOverviews(
     /* -------------------------------------------------------------------- */
     double dfPixelsProcessed = 0.0;
 
+    CPLStringList aosOptions(papszOptions);
+    aosOptions.SetNameValue("CASCADING", "YES");
     for (int i = 0; i < nOverviews; ++i)
     {
         GDALRasterBand *poBaseBand = poSrcBand;
@@ -4577,7 +4579,7 @@ static CPLErr GDALRegenerateCascadingOverviews(
             poBaseBand, 1,
             reinterpret_cast<GDALRasterBandH *>(papoOvrBands) + i,
             pszResampling, GDALScaledProgress, pScaledProgressData,
-            papszOptions);
+            aosOptions.List());
         GDALDestroyScaledProgress(pScaledProgressData);
 
         if (eErr != CE_None)
@@ -4896,6 +4898,26 @@ CPLErr GDALRegenerateOverviewsEx(GDALRasterBandH hSrcBand, int nOverviewCount,
         }
     }
 
+    int nHasNoData = 0;
+    const double dfNoDataValue = poSrcBand->GetNoDataValue(&nHasNoData);
+    const bool bHasNoData = CPL_TO_BOOL(nHasNoData);
+    const bool bPropagateNoData =
+        CPLTestBool(CPLGetConfigOption("GDAL_OVR_PROPAGATE_NODATA", "NO"));
+
+    if (poSrcBand->GetBand() == 1 && bUseNoDataMask &&
+        CSLFetchNameValue(papszOptions, "CASCADING") == nullptr)
+    {
+        std::string osDetailMessage;
+        if (poSrcBand->HasConflictingMaskSources(&osDetailMessage, false))
+        {
+            CPLError(
+                CE_Warning, CPLE_AppDefined, "%s%s", osDetailMessage.c_str(),
+                bHasNoData
+                    ? "Only the nodata value will be taken into account."
+                    : "Only the first listed one will be taken into account.");
+        }
+    }
+
     /* -------------------------------------------------------------------- */
     /*      If we are operating on multiple overviews, and using            */
     /*      averaging, lets do them in cascading order to reduce the        */
@@ -5027,12 +5049,6 @@ CPLErr GDALRegenerateOverviewsEx(GDALRasterBandH hSrcBand, int nOverviewCount,
         }
     }
 
-    int nHasNoData = 0;
-    const double dfNoDataValue = poSrcBand->GetNoDataValue(&nHasNoData);
-    const bool bHasNoData = CPL_TO_BOOL(nHasNoData);
-    const bool bPropagateNoData =
-        CPLTestBool(CPLGetConfigOption("GDAL_OVR_PROPAGATE_NODATA", "NO"));
-
     // Structure describing a resampling job
     struct OvrJob
     {
@@ -5683,6 +5699,17 @@ CPLErr GDALRegenerateOverviewsMultiBand(
             papoSrcBands[iBand]->GetNoDataValue(&nHasNoData);
         abHasNoData[iBand] = CPL_TO_BOOL(nHasNoData);
     }
+
+    std::string osDetailMessage;
+    if (bUseNoDataMask &&
+        papoSrcBands[0]->HasConflictingMaskSources(&osDetailMessage, false))
+    {
+        CPLError(CE_Warning, CPLE_AppDefined, "%s%s", osDetailMessage.c_str(),
+                 abHasNoData[0]
+                     ? "Only the nodata value will be taken into account."
+                     : "Only the first listed one will be taken into account.");
+    }
+
     const bool bPropagateNoData =
         CPLTestBool(CPLGetConfigOption("GDAL_OVR_PROPAGATE_NODATA", "NO"));