Skip to content

Commit f5812d7

Browse files
abradley60abradley60
authored
Merge pull request #96 from GeoscienceAustralia/bugfix/special_xml_tags
handling special xml tags
2 parents d9c1873 + c699215 commit f5812d7

8 files changed

Lines changed: 419 additions & 196 deletions

File tree

‎sar_pipeline/aws/cli.py‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -660,12 +660,16 @@ def make_rtc_opera_stac_and_upload_bursts(
660660
f"Making STAC metadata for burst {i+1} of {len(burst_folders)} : {burst_folder}"
661661
)
662662

663-
logger.info(f"Renaming all files so 'v' is not in the product version number ")
663+
logger.info(
664+
f"Renaming all files so 'v' is not in the product version number, and version is '-' separated "
665+
)
664666
for product_file in burst_folder.iterdir():
665667
if product_file.is_file():
666-
new_path = product_file.with_name(
667-
re.sub(r"v(?=\d)", "", product_file.name)
668-
)
668+
# step 1: remove 'v' before version numbers
669+
name = re.sub(r"v(?=\d)", "", product_file.name)
670+
# step 2: replace version pattern digits.digits.digits → digits-digits-digits
671+
name = re.sub(r"(\d+)\.(\d+)\.(\d+)", r"\1-\2-\3", name)
672+
new_path = product_file.with_name(name)
669673
if new_path != product_file:
670674
logger.info(f"Renaming: {product_file.name} -> {new_path.name}")
671675
product_file.rename(new_path)
@@ -761,9 +765,18 @@ def make_rtc_opera_stac_and_upload_bursts(
761765
# create the XML file from the existing metadata files
762766
if xml_filepath:
763767
logger.info("Creating the XML file from the stac and .h5 metadata files")
764-
XML = XMLMapper(stac_path=stac_filepath, h5_path=burst_h5_filepath)
768+
XML = XMLMapper(
769+
stac_path=stac_filepath,
770+
h5_path=burst_h5_filepath,
771+
polarisations=burst_stac_manager.polarisations,
772+
backscatter_convention=backscatter_convention,
773+
)
765774
logger.info("Populating the XML template using the mapping file")
766775
XML.populate_xml()
776+
logger.info(
777+
f"Adding special mappings to XML template. e.g. multiple backscatter pols"
778+
)
779+
XML.populate_special_xml_mappings()
767780
logger.info(f"Saving XML file to : {xml_filepath}")
768781
XML.save_xml(xml_filepath)
769782

@@ -773,7 +786,7 @@ def make_rtc_opera_stac_and_upload_bursts(
773786
else:
774787
# re-check that the files don't already exist in S3. This will help protect against
775788
# simultaneous runs of the same product. E.g. the product did not exist at the
776-
# start of the run and was created by another process during this run
789+
# start of the run and was created by another process during this runtime
777790
logger.info(
778791
f"Checking if product already exist before uploading for burst : {burst_stac_manager.burst_id}"
779792
)

‎sar_pipeline/aws/metadata/stac.py‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ def add_properties_from_h5(self):
258258
# add product stac extension properties
259259
self.item.properties["product:type"] = self.product
260260
# remove timeliness as not required. May re-add if approach is determined.
261-
# self.item.properties["product:timeliness"] = "TODO"
261+
# self.item.properties["product:timeliness"] = ""
262262
# self.item.properties["product:timeliness_category"] = (
263263
# self._get_product_timeliness_category(self.start_dt, self.processed_dt)
264264
# )
@@ -272,9 +272,9 @@ def add_properties_from_h5(self):
272272
# add projection (proj) stac extension properties
273273
self.item.properties["proj:code"] = f"EPSG:{self.projection_epsg}"
274274
self.item.properties["proj:bbox"] = self.h5.search_value("boundingBox")
275-
self.item.properties["proj:wkt2"] = pyproj.CRS.from_epsg(
276-
self.projection_epsg
277-
).to_wkt()
275+
# self.item.properties["proj:wkt2"] = pyproj.CRS.from_epsg(
276+
# self.projection_epsg
277+
# ).to_wkt() # causing issues in explorer, value set in XML
278278

279279
# add the sar stac extension properties
280280
self.item.properties["sar:frequency_band"] = self.h5.search_value("radarBand")
@@ -330,7 +330,9 @@ def add_properties_from_h5(self):
330330
}
331331

332332
# proposed sarard stac extension properties
333-
self.item.properties["sarard:source_id"] = self.h5.search_value("l1SlcGranules")
333+
self.item.properties["sarard:source_id"] = self.h5.search_value(
334+
"l1SlcGranules"
335+
)[0]
334336
self.item.properties["sarard:source_geometry"] = "slant range"
335337
self.item.properties["sarard:scene_id"] = self.h5.search_value("l1SlcGranules")[
336338
0
@@ -344,9 +346,9 @@ def add_properties_from_h5(self):
344346
self.item.properties["sarard:far_range_incidence_angle"] = (
345347
self.h5.search_value("farRangeIncidenceAngle")
346348
)
347-
self.item.properties["sarard:orbit_files"] = self.h5.search_value(
348-
"orbitFiles"
349-
) # Link to a file containing the orbit state vectors.
349+
self.item.properties["sarard:orbit_file"] = self.h5.search_value("orbitFiles")[
350+
0
351+
] # Link to a file containing the orbit state vectors.
350352
self.item.properties["sarard:UL_longitude"] = self.bbox_4326[0] # min_lon
351353
self.item.properties["sarard:UL_latitude"] = self.bbox_4326[3] # max_lat
352354
self.item.properties["sarard:LR_longitude"] = self.bbox_4326[2] # max_lon

‎sar_pipeline/aws/metadata/templates/s1nrb.xml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -104,16 +104,16 @@
104104
<DemSource type="URL"></DemSource>
105105
</AuxiliaryData>
106106
<ProductSampleSpacing>
107-
<ProductColumnSpacing units="meter">
107+
<ProductColumnSpacing units="m">
108108
</ProductColumnSpacing>
109-
<ProductRowSpacing units="meter">
109+
<ProductRowSpacing units="m">
110110
</ProductRowSpacing>
111111
</ProductSampleSpacing>
112112
<ProductENL>
113113
</ProductENL>
114114
<ProductResolution>
115-
<ProductResolutionColumn unit="meter"></ProductResolutionColumn>
116-
<ProductResolutionRow unit="meter"></ProductResolutionRow>
115+
<ProductResolutionColumn unit="m"></ProductResolutionColumn>
116+
<ProductResolutionRow unit="m"></ProductResolutionRow>
117117
</ProductResolution>
118118
<Filtering>
119119
<FilterApplied>
@@ -149,7 +149,7 @@
149149
</CoordinateReferenceSystem>
150150
<PerPixelMetadata>
151151
<DataMask>
152-
<FileName>
152+
<FileName type="URL">
153153
</FileName>
154154
<SampleType>
155155
</SampleType>
@@ -179,7 +179,7 @@
179179
</BitValues>
180180
</DataMask>
181181
<LocalIncAngle>
182-
<FileName>
182+
<FileName type="URL">
183183
</FileName>
184184
<SampleType units="deg">
185185
</SampleType>
@@ -193,7 +193,7 @@
193193
</ByteOrder>
194194
</LocalIncAngle>
195195
<EllipsoidIncAngle>
196-
<FileName>
196+
<FileName type="URL">
197197
</FileName>
198198
<SampleType units="deg">
199199
</SampleType>
@@ -209,7 +209,7 @@
209209
</EllipsoidName>
210210
</EllipsoidIncAngle>
211211
<GammaToSigmaRatio>
212-
<FileName>
212+
<FileName type="URL">
213213
</FileName>
214214
<SampleType units="Ratio">
215215
</SampleType>
@@ -232,7 +232,7 @@
232232
</BackscatterConversionEq>
233233
<Polarization>
234234
</Polarization>
235-
<FileName>
235+
<FileName type="URL">
236236
</FileName>
237237
<DataFormat>
238238
</DataFormat>
@@ -256,6 +256,8 @@
256256
<GeometricCorrection>
257257
<GeoCorrAlgorithm type="DOI">
258258
</GeoCorrAlgorithm>
259+
<GriddingConvention>
260+
</GriddingConvention>
259261
<DigitalElevationModel dem="Surface">
260262
<DEMReference type="URL">
261263
</DEMReference>
@@ -275,7 +277,5 @@
275277
</GeoAccuracyReference>
276278
</GeoCorrAccuracy>
277279
</GeometricCorrection>
278-
<GridName>
279-
</GridName>
280280
</CEOS-ARDProductAttributes>
281281
</Product>

0 commit comments

Comments
 (0)