Skip to content

Commit 2c989b6

Browse files
committed
gdal CLI: add 'gdal mdim pipeline'
This is minimal support. It converts 'info' to be compatible, and adds a minimalistic 'reproject' (step or standalone) ``` $ gdal mdim pipeline ! read in.nc ! reproject --output-crs=EPSG:32632 ! write out.zarr --overwrite $ gdal mdim reproject --output-crs=EPSG:32632 in.nc out.zarr --overwrite ```
1 parent 152a869 commit 2c989b6

35 files changed

Lines changed: 1852 additions & 84 deletions

.github/workflows/ubuntu_26.04/expected_gdalinfo_formats.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:write from scratch, u:update,
134134
STACTA -raster- (rovs): Spatio-Temporal Asset Catalog Tiled Assets (*.json)
135135
STACIT -raster- (rovs): Spatio-Temporal Asset Catalog Items
136136
JPEGXL -raster- (rwv): JPEG-XL (*.jxl)
137-
GDALG -raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
137+
GDALG -raster,multidimensional raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
138138
E57 -raster- (rovs): ASTM E57 3D file format (image part) (*.e57)
139139
GPKG -raster,vector- (rw+uvs): GeoPackage (*.gpkg, *.gpkg.zip)
140140
SQLite -raster,vector- (rw+uvs): SQLite / Spatialite / RasterLite2 (*.sqlite, *.db)

.github/workflows/ubuntu_26.04/expected_ogrinfo_formats.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:write from scratch, u:update,
1212
BAG -raster,multidimensional raster,vector- (rw+v): Bathymetry Attributed Grid (*.bag)
1313
EEDA -vector- (ro): Earth Engine Data API
1414
OGCAPI -raster,vector- (rov): OGCAPI
15-
GDALG -raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
15+
GDALG -raster,multidimensional raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
1616
ESRI Shapefile -vector- (rw+uv): ESRI Shapefile (*.shp, *.dbf, *.shz, *.shp.zip)
1717
MapInfo File -vector- (rw+uv): MapInfo File (*.tab, *.mif, *.mid)
1818
UK .NTF -vector- (rov): UK .NTF

.github/workflows/windows_conda_expected_gdalinfo_formats.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:write from scratch, u:update,
130130
OGCAPI -raster,vector- (rov): OGCAPI
131131
STACTA -raster- (rovs): Spatio-Temporal Asset Catalog Tiled Assets (*.json)
132132
STACIT -raster- (rovs): Spatio-Temporal Asset Catalog Items
133-
GDALG -raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
133+
GDALG -raster,multidimensional raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
134134
E57 -raster- (rovs): ASTM E57 3D file format (image part) (*.e57)
135135
NSIDCbin -raster- (rov): NSIDC Sea Ice Concentrations binary (.bin) (*.bin)
136136
CPHD -multidimensional raster- (ro): Compensated Phase History Data Reader

.github/workflows/windows_conda_expected_ogrinfo_formats.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:write from scratch, u:update,
1111
BAG -raster,multidimensional raster,vector- (rw+v): Bathymetry Attributed Grid (*.bag)
1212
EEDA -vector- (ro): Earth Engine Data API
1313
OGCAPI -raster,vector- (rov): OGCAPI
14-
GDALG -raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
14+
GDALG -raster,multidimensional raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
1515
TileDB -raster,multidimensional raster,vector- (rw+uvs): TileDB
1616
ESRI Shapefile -vector- (rw+uv): ESRI Shapefile (*.shp, *.dbf, *.shz, *.shp.zip)
1717
MapInfo File -vector- (rw+uv): MapInfo File (*.tab, *.mif, *.mid)

apps/CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ target_sources(appslib PRIVATE
4444
gdalalg_mdim_info.cpp
4545
gdalalg_mdim_convert.cpp
4646
gdalalg_mdim_mosaic.cpp
47+
gdalalg_mdim_pipeline.cpp
48+
gdalalg_mdim_read.cpp
49+
gdalalg_mdim_reproject.cpp
50+
gdalalg_mdim_write.cpp
4751
gdalalg_pipeline.cpp
4852
gdalalg_raster.cpp
4953
gdalalg_raster_info.cpp

apps/gdalalg_abstract_pipeline.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ class GDALPipelineStepAlgorithm /* non final */ : public GDALAlgorithm
302302
friend class GDALPipelineAlgorithm;
303303
friend class GDALRasterPipelineAlgorithm;
304304
friend class GDALVectorPipelineAlgorithm;
305+
friend class GDALMdimPipelineAlgorithm;
305306
friend class GDALAbstractPipelineAlgorithm;
306307

307308
virtual bool CanBeFirstStep() const
@@ -398,6 +399,11 @@ class GDALPipelineStepAlgorithm /* non final */ : public GDALAlgorithm
398399
void AddOutputLayerNameArg(bool hiddenForCLI,
399400
bool shortNameOutputLayerAllowed);
400401

402+
void AddMdimInputArgs(bool openForMixedRasterVector, bool hiddenForCLI,
403+
bool acceptRaster);
404+
void AddMdimOutputArgs(bool hiddenForCLI);
405+
void AddMdimHiddenInputDatasetArg();
406+
401407
private:
402408
bool RunImpl(GDALProgressFunc pfnProgress, void *pProgressData) override;
403409
GDALAlgorithm::ProcessGDALGOutputRet ProcessGDALGOutput() override;
@@ -435,6 +441,7 @@ class GDALAbstractPipelineAlgorithm CPL_NON_FINAL
435441

436442
static constexpr const char *RASTER_SUFFIX = "-raster";
437443
static constexpr const char *VECTOR_SUFFIX = "-vector";
444+
static constexpr const char *MULTIDIM_SUFFIX = "-multidim";
438445

439446
protected:
440447
friend class GDALTeeStepAlgorithmAbstract;
@@ -467,6 +474,7 @@ class GDALAbstractPipelineAlgorithm CPL_NON_FINAL
467474
friend class GDALPipelineAlgorithm;
468475
friend class GDALRasterPipelineAlgorithm;
469476
friend class GDALVectorPipelineAlgorithm;
477+
friend class GDALMdimPipelineAlgorithm;
470478

471479
std::vector<std::unique_ptr<GDALPipelineStepAlgorithm>> m_steps{};
472480

apps/gdalalg_mdim.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
#include "gdalalg_mdim_info.h"
1818
#include "gdalalg_mdim_convert.h"
1919
#include "gdalalg_mdim_mosaic.h"
20+
#include "gdalalg_mdim_pipeline.h"
21+
#include "gdalalg_mdim_reproject.h"
2022

2123
#include "gdal_priv.h"
2224

@@ -37,9 +39,11 @@ GDALMdimAlgorithm::GDALMdimAlgorithm()
3739

3840
AddOutputStringArg(&m_output);
3941

40-
RegisterSubAlgorithm<GDALMdimInfoAlgorithm>();
42+
RegisterSubAlgorithm<GDALMdimInfoAlgorithmStandalone>();
4143
RegisterSubAlgorithm<GDALMdimConvertAlgorithm>();
4244
RegisterSubAlgorithm<GDALMdimMosaicAlgorithm>();
45+
RegisterSubAlgorithm<GDALMdimPipelineAlgorithm>();
46+
RegisterSubAlgorithm<GDALMdimReprojectAlgorithmStandalone>();
4347
}
4448

4549
bool GDALMdimAlgorithm::RunImpl(GDALProgressFunc, void *)

apps/gdalalg_mdim_convert.cpp

Lines changed: 50 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -27,21 +27,15 @@
2727
/************************************************************************/
2828

2929
GDALMdimConvertAlgorithm::GDALMdimConvertAlgorithm()
30-
: GDALAlgorithm(NAME, DESCRIPTION, HELP_URL)
30+
: GDALMdimPipelineStepAlgorithm(NAME, DESCRIPTION, HELP_URL,
31+
ConstructorOptions()
32+
.SetStandaloneStep(true)
33+
.SetInputDatasetMaxCount(1)
34+
.SetAddDefaultArguments(false))
3135
{
36+
AddMdimInputArgs(false, false, /* acceptRaster = */ true);
3237
AddProgressArg();
33-
AddOutputFormatArg(&m_outputFormat)
34-
.AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
35-
{GDAL_DCAP_CREATE_MULTIDIMENSIONAL});
36-
AddOpenOptionsArg(&m_openOptions);
37-
AddInputFormatsArg(&m_inputFormats)
38-
.AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
39-
{GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER});
40-
AddInputDatasetArg(&m_inputDataset,
41-
GDAL_OF_RASTER | GDAL_OF_MULTIDIM_RASTER);
42-
AddOutputDatasetArg(&m_outputDataset, GDAL_OF_MULTIDIM_RASTER);
43-
AddCreationOptionsArg(&m_creationOptions);
44-
AddOverwriteArg(&m_overwrite);
38+
AddMdimOutputArgs(false);
4539

4640
{
4741
auto &arg = AddArg("array", 0,
@@ -55,14 +49,18 @@ GDALMdimConvertAlgorithm::GDALMdimConvertAlgorithm()
5549
[this](const std::string &)
5650
{
5751
std::vector<std::string> ret;
58-
59-
if (auto poDS = std::unique_ptr<GDALDataset>(GDALDataset::Open(
60-
m_inputDataset.GetName().c_str(),
61-
GDAL_OF_MULTIDIM_RASTER, nullptr, nullptr, nullptr)))
52+
if (m_inputDataset.size() == 1)
6253
{
63-
if (auto poRG = poDS->GetRootGroup())
54+
if (auto poDS =
55+
std::unique_ptr<GDALDataset>(GDALDataset::Open(
56+
m_inputDataset[0].GetName().c_str(),
57+
GDAL_OF_MULTIDIM_RASTER, nullptr, nullptr,
58+
nullptr)))
6459
{
65-
ret = poRG->GetMDArrayFullNamesRecursive();
60+
if (auto poRG = poDS->GetRootGroup())
61+
{
62+
ret = poRG->GetMDArrayFullNamesRecursive();
63+
}
6664
}
6765
}
6866

@@ -84,17 +82,22 @@ GDALMdimConvertAlgorithm::GDALMdimConvertAlgorithm()
8482
[this](const std::string &currentValue)
8583
{
8684
std::vector<std::string> ret;
87-
88-
if (auto poDS = std::unique_ptr<GDALDataset>(GDALDataset::Open(
89-
m_inputDataset.GetName().c_str(),
90-
GDAL_OF_MULTIDIM_RASTER, nullptr, nullptr, nullptr)))
85+
if (m_inputDataset.size() == 1)
9186
{
92-
if (auto poDriver = poDS->GetDriver())
87+
if (auto poDS =
88+
std::unique_ptr<GDALDataset>(GDALDataset::Open(
89+
m_inputDataset[0].GetName().c_str(),
90+
GDAL_OF_MULTIDIM_RASTER, nullptr, nullptr,
91+
nullptr)))
9392
{
94-
if (const char *pszXML = poDriver->GetMetadataItem(
95-
GDAL_DMD_MULTIDIM_ARRAY_OPENOPTIONLIST))
93+
if (auto poDriver = poDS->GetDriver())
9694
{
97-
AddOptionsSuggestions(pszXML, 0, currentValue, ret);
95+
if (const char *pszXML = poDriver->GetMetadataItem(
96+
GDAL_DMD_MULTIDIM_ARRAY_OPENOPTIONLIST))
97+
{
98+
AddOptionsSuggestions(pszXML, 0, currentValue,
99+
ret);
100+
}
98101
}
99102
}
100103
}
@@ -129,14 +132,28 @@ GDALMdimConvertAlgorithm::GDALMdimConvertAlgorithm()
129132
bool GDALMdimConvertAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
130133
void *pProgressData)
131134
{
132-
CPLAssert(m_inputDataset.GetDatasetRef());
135+
GDALPipelineStepRunContext stepCtxt;
136+
stepCtxt.m_pfnProgress = pfnProgress;
137+
stepCtxt.m_pProgressData = pProgressData;
138+
return RunPreStepPipelineValidations() && RunStep(stepCtxt);
139+
}
140+
141+
/************************************************************************/
142+
/* GDALMdimConvertAlgorithm::RunStep() */
143+
/************************************************************************/
144+
145+
bool GDALMdimConvertAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
146+
{
147+
auto poSrcDS = m_inputDataset[0].GetDatasetRef();
148+
CPLAssert(poSrcDS);
149+
133150
CPLAssert(!m_outputDataset.GetDatasetRef());
134151

135152
CPLStringList aosOptions;
136-
if (!m_outputFormat.empty())
153+
if (!m_format.empty())
137154
{
138155
aosOptions.AddString("-of");
139-
aosOptions.AddString(m_outputFormat.c_str());
156+
aosOptions.AddString(m_format.c_str());
140157
}
141158
if (m_overwrite)
142159
{
@@ -192,10 +209,12 @@ bool GDALMdimConvertAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
192209

193210
GDALMultiDimTranslateOptions *psOptions =
194211
GDALMultiDimTranslateOptionsNew(aosOptions.List(), nullptr);
212+
auto pfnProgress = ctxt.m_pfnProgress;
213+
auto pProgressData = ctxt.m_pProgressData;
195214
GDALMultiDimTranslateOptionsSetProgress(psOptions, pfnProgress,
196215
pProgressData);
197216

198-
auto hSrcDS = GDALDataset::ToHandle(m_inputDataset.GetDatasetRef());
217+
auto hSrcDS = GDALDataset::ToHandle(poSrcDS);
199218
auto poOutDS = std::unique_ptr<GDALDataset>(GDALDataset::FromHandle(
200219
GDALMultiDimTranslate(m_outputDataset.GetName().c_str(), nullptr, 1,
201220
&hSrcDS, psOptions, nullptr)));

apps/gdalalg_mdim_convert.h

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,15 @@
1313
#ifndef GDALALG_MDIM_CONVERT_INCLUDED
1414
#define GDALALG_MDIM_CONVERT_INCLUDED
1515

16-
#include "gdalalgorithm.h"
16+
#include "gdalalg_mdim_pipeline.h"
1717

1818
//! @cond Doxygen_Suppress
1919

2020
/************************************************************************/
2121
/* GDALMdimConvertAlgorithm */
2222
/************************************************************************/
2323

24-
class GDALMdimConvertAlgorithm final : public GDALAlgorithm
24+
class GDALMdimConvertAlgorithm final : public GDALMdimPipelineStepAlgorithm
2525
{
2626
public:
2727
static constexpr const char *NAME = "convert";
@@ -32,15 +32,9 @@ class GDALMdimConvertAlgorithm final : public GDALAlgorithm
3232
explicit GDALMdimConvertAlgorithm();
3333

3434
private:
35+
bool RunStep(GDALPipelineStepRunContext &ctxt) override;
3536
bool RunImpl(GDALProgressFunc pfnProgress, void *pProgressData) override;
3637

37-
std::string m_outputFormat{};
38-
GDALArgDatasetValue m_inputDataset{};
39-
std::vector<std::string> m_openOptions{};
40-
std::vector<std::string> m_inputFormats{};
41-
GDALArgDatasetValue m_outputDataset{};
42-
std::vector<std::string> m_creationOptions{};
43-
bool m_overwrite = false;
4438
bool m_strict = false;
4539
std::vector<std::string> m_arrays{};
4640
std::vector<std::string> m_arrayOptions{};

apps/gdalalg_mdim_info.cpp

Lines changed: 43 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -26,17 +26,30 @@
2626
/* GDALMdimInfoAlgorithm::GDALMdimInfoAlgorithm() */
2727
/************************************************************************/
2828

29-
GDALMdimInfoAlgorithm::GDALMdimInfoAlgorithm()
30-
: GDALAlgorithm(NAME, DESCRIPTION, HELP_URL)
29+
GDALMdimInfoAlgorithm::GDALMdimInfoAlgorithm(bool standaloneStep,
30+
bool openForMixedRasterVector)
31+
: GDALMdimPipelineStepAlgorithm(
32+
NAME, DESCRIPTION, HELP_URL,
33+
ConstructorOptions()
34+
.SetStandaloneStep(standaloneStep)
35+
.SetInputDatasetMaxCount(1)
36+
.SetAddDefaultArguments(false)
37+
.SetInputDatasetHelpMsg(_("Input multidimensional dataset"))
38+
.SetInputDatasetAlias("dataset"))
3139
{
40+
if (standaloneStep)
41+
{
42+
AddMdimInputArgs(openForMixedRasterVector,
43+
/* hiddenForCLI = */ false,
44+
/* acceptRaster = */ false);
45+
}
46+
else
47+
{
48+
AddMdimHiddenInputDatasetArg();
49+
}
50+
3251
AddOutputFormatArg(&m_format).SetHidden().SetDefault("json").SetChoices(
3352
"json", "text");
34-
AddOpenOptionsArg(&m_openOptions);
35-
AddInputFormatsArg(&m_inputFormats)
36-
.AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
37-
{GDAL_DCAP_MULTIDIM_RASTER});
38-
AddInputDatasetArg(&m_dataset, GDAL_OF_MULTIDIM_RASTER).AddAlias("dataset");
39-
AddOutputStringArg(&m_output);
4053
AddArg("summary", 0,
4154
_("Report only group and array hierarchy, without detailed "
4255
"information on attributes or dimensions."),
@@ -68,17 +81,22 @@ GDALMdimInfoAlgorithm::GDALMdimInfoAlgorithm()
6881
[this](const std::string &currentValue)
6982
{
7083
std::vector<std::string> ret;
71-
72-
if (auto poDS = std::unique_ptr<GDALDataset>(GDALDataset::Open(
73-
m_dataset.GetName().c_str(), GDAL_OF_MULTIDIM_RASTER,
74-
nullptr, nullptr, nullptr)))
84+
if (m_inputDataset.size() == 1)
7585
{
76-
if (auto poDriver = poDS->GetDriver())
86+
if (auto poDS =
87+
std::unique_ptr<GDALDataset>(GDALDataset::Open(
88+
m_inputDataset[0].GetName().c_str(),
89+
GDAL_OF_MULTIDIM_RASTER, nullptr, nullptr,
90+
nullptr)))
7791
{
78-
if (const char *pszXML = poDriver->GetMetadataItem(
79-
GDAL_DMD_MULTIDIM_ARRAY_OPENOPTIONLIST))
92+
if (auto poDriver = poDS->GetDriver())
8093
{
81-
AddOptionsSuggestions(pszXML, 0, currentValue, ret);
94+
if (const char *pszXML = poDriver->GetMetadataItem(
95+
GDAL_DMD_MULTIDIM_ARRAY_OPENOPTIONLIST))
96+
{
97+
AddOptionsSuggestions(pszXML, 0, currentValue,
98+
ret);
99+
}
82100
}
83101
}
84102
}
@@ -88,16 +106,19 @@ GDALMdimInfoAlgorithm::GDALMdimInfoAlgorithm()
88106
}
89107
AddArg("stats", 0, _("Read and display image statistics."), &m_stats);
90108

109+
AddOutputStringArg(&m_output);
91110
AddStdoutArg(&m_stdout);
92111
}
93112

94113
/************************************************************************/
95-
/* GDALMdimInfoAlgorithm::RunImpl() */
114+
/* GDALMdimInfoAlgorithm::RunStep() */
96115
/************************************************************************/
97116

98-
bool GDALMdimInfoAlgorithm::RunImpl(GDALProgressFunc, void *)
117+
bool GDALMdimInfoAlgorithm::RunStep(GDALPipelineStepRunContext &)
99118
{
100-
CPLAssert(m_dataset.GetDatasetRef());
119+
CPLAssert(m_inputDataset.size() == 1);
120+
auto poSrcDS = m_inputDataset[0].GetDatasetRef();
121+
CPLAssert(poSrcDS);
101122

102123
CPLStringList aosOptions;
103124

@@ -125,7 +146,7 @@ bool GDALMdimInfoAlgorithm::RunImpl(GDALProgressFunc, void *)
125146
aosOptions.AddString(opt.c_str());
126147
}
127148

128-
GDALDatasetH hDS = GDALDataset::ToHandle(m_dataset.GetDatasetRef());
149+
GDALDatasetH hDS = GDALDataset::ToHandle(poSrcDS);
129150
GDALMultiDimInfoOptions *psOptions =
130151
GDALMultiDimInfoOptionsNew(aosOptions.List(), nullptr);
131152
char *ret = GDALMultiDimInfo(hDS, psOptions);
@@ -140,4 +161,6 @@ bool GDALMdimInfoAlgorithm::RunImpl(GDALProgressFunc, void *)
140161
return bOK;
141162
}
142163

164+
GDALMdimInfoAlgorithmStandalone::~GDALMdimInfoAlgorithmStandalone() = default;
165+
143166
//! @endcond

0 commit comments

Comments
 (0)