Skip to content

Commit a9a85c5

Browse files
botanicalanayeaye
botanical
and
anayeaye
authored
feat: update workflow to process all added, changed, modified, and renamed dataset-config files (#321)
* feat: update workflow to process all added, changed, modified, and renamed dataset-config files * fix: add test file for testing changes * fix: added test file to test A * fix: attempt formatting comment string * fix: use br instead of n * fix: alternative formatting for comment * fix: update formatting of comment * Update ingestion-data/staging/dataset-config/test-added.json Co-authored-by: Alexandra Kirk <[email protected]> --------- Co-authored-by: Alexandra Kirk <[email protected]>
1 parent 71d4d93 commit a9a85c5

File tree

3 files changed

+95
-20
lines changed

3 files changed

+95
-20
lines changed

‎.github/workflows/pr.yml

+17-18
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
outputs:
3232
published_collections: ${{ steps.publish-collections.outputs.success_collections }}
3333
comment_id: ${{ steps.init-comment.outputs.COMMENT_ID }}
34-
has_new_files_to_promote: ${{ steps.changed-files.outputs.added_files_count != 0 }}
34+
has_new_files_to_promote: ${{ steps.changed-files.outputs.all_changed_files_count != 0 }}
3535
steps:
3636
- uses: actions/checkout@v4
3737

@@ -66,27 +66,26 @@ jobs:
6666
6767
echo "COMMENT_ID=$COMMENT_ID" >> $GITHUB_OUTPUT
6868
69-
# Find only the newly added files
70-
# Only .json files
7169
# The files are outputted to GITHUB_OUTPUT, which can be used in subsequent steps
72-
- name: Get newly added files
70+
- name: Get newly added and changed files
7371
id: changed-files
7472
uses: tj-actions/changed-files@v45
7573
with:
7674
files: ingestion-data/staging/dataset-config/**.json
7775

78-
- name: List all newly added files
76+
# ACMR - Added, Copied, Modified, Renamed
77+
- name: List all newly added and changed files
7978
env:
80-
ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
81-
ADDED_FILES_COUNT: $${{ steps.changed-files.outputs.added_files_count}}
79+
ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
80+
ALL_CHANGED_FILES_COUNT: $${{ steps.changed-files.outputs.all_changed_files_count}}
8281
run: |
83-
for file in ${ADDED_FILES}; do
82+
for file in ${ALL_CHANGED_FILES}; do
8483
echo "$file was added"
8584
done
86-
echo "added_files_count: ${ADDED_FILES_COUNT}"
85+
echo "all_changed_files_count: ${ALL_CHANGED_FILES_COUNT}"
8786
8887
- name: Exit early if no new files
89-
if: ${{ steps.changed-files.outputs.added_files_count == 0 }}
88+
if: ${{ steps.changed-files.outputs.all_changed_files_count == 0 }}
9089
run: |
9190
echo "🕵️ No new files found. Exiting workflow early."
9291
exit 0
@@ -97,10 +96,10 @@ jobs:
9796
# If none of the requests are successful, workflow fails
9897
# Updates the PR comment with status of collection publication
9998
- name: Publish all newly added collections to staging
100-
if: ${{ steps.changed-files.outputs.added_files_count != 0 }}
99+
if: ${{ steps.changed-files.outputs.all_changed_files_count != 0 }}
101100
id: publish-collections
102101
env:
103-
ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
102+
ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
104103
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
105104
COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
106105
STAGING_SM2A_ADMIN_USERNAME: ${{ secrets.STAGING_SM2A_ADMIN_USERNAME }}
@@ -113,9 +112,9 @@ jobs:
113112
# Track successful publications
114113
all_failed=true
115114
declare -a success_collections=()
116-
status_message="### Collection Publication Status"
115+
status_message=""
117116
118-
for file in ${ADDED_FILES}; do
117+
for file in ${ALL_CHANGED_FILES}; do
119118
echo $file
120119
if [ -f "$file" ]; then
121120
dataset_config=$(jq '.' "$file")
@@ -131,12 +130,12 @@ jobs:
131130
# Update status message based on response code
132131
if [[ $status_code -eq 200 ]] || [[ $status_code -eq 201 ]]; then
133132
echo "$collection_id successfully published ✅"
134-
status_message+="- **$collection_id**: Successfully published ✅"
133+
status_message+="➡️ **$collection_id**: Successfully published ✅ <br>"
135134
success_collections+=("$file")
136135
all_failed=false
137136
else
138137
echo "$collection_id failed to publish ❌"
139-
status_message+="- **$collection_id**: Failed to publish. Error code $status_code. ❌"
138+
status_message+="➡️ **$collection_id**: Failed to publish. Error code $status_code. ❌<br>"
140139
fi
141140
else
142141
echo "File $file does not exist"
@@ -156,7 +155,7 @@ jobs:
156155
# Update PR comment
157156
CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
158157
UPDATED_BODY="$CURRENT_BODY
159-
158+
### Collection Publication Status
160159
$status_message"
161160
gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
162161
@@ -186,7 +185,7 @@ jobs:
186185
# If the workflow fails at any point, the PR comment will be updated
187186
188187
- name: Update PR comment if no new files
189-
if: ${{ steps.changed-files.outputs.added_files_count == 0 }}
188+
if: ${{ steps.changed-files.outputs.all_changed_files_count == 0 }}
190189
env:
191190
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
192191
COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}

‎ingestion-data/staging/dataset-config/test-added.json

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
{
2+
"collection": "campfire-nlcd-TEST-ADDED",
3+
"title": "Camp Fire Domain: Land Cover TEST ADDED",
4+
"description": "NLCD-derived land use-land cover for 2016 over the Camp Fire burn scar domain. Displays colors 0-255 from a lookup table for land cover type (ex: reds = urbanized).",
5+
"type": "Collection",
6+
"spatial_extent": {
7+
"xmin": -125,
8+
"ymin": 40,
9+
"xmax": -122,
10+
"ymax": 45
11+
},
12+
"temporal_extent": {
13+
"startdate": "2016-01-01T00:00:00Z",
14+
"enddate": "2019-12-31T00:00:00Z"
15+
},
16+
"license": "CC0-1.0",
17+
"stac_version": "1.0.0",
18+
"discovery_items": [
19+
{
20+
"prefix": "campfire-data/",
21+
"bucket": "veda-data-store-staging",
22+
"filename_regex": "^(.*)NLCD_20.*_cog.tif$",
23+
"discovery": "s3",
24+
"upload": false,
25+
"cogify": false,
26+
"dry_run": false
27+
}
28+
],
29+
"providers": [
30+
{
31+
"name": "NASA VEDA",
32+
"url": "https://www.earthdata.nasa.gov/dashboard/",
33+
"roles": [
34+
"host"
35+
]
36+
}
37+
],
38+
"item_assets": {
39+
"cog_default": {
40+
"type": "image/tiff; application=geotiff; profile=cloud-optimized",
41+
"roles": [
42+
"data",
43+
"layer"
44+
],
45+
"title": "Default COG Layer",
46+
"description": "Cloud optimized default layer to display on map"
47+
}
48+
},
49+
"assets": {
50+
"thumbnail": {
51+
"title": "Thumbnail",
52+
"description": "Photo by [Mike Newbry](https://unsplash.com/photos/DwtX9mMHBJ0) (Engulfed hillside in California, 2021)",
53+
"href": "https://thumbnails.openveda.cloud/camp-fire-background.jpg",
54+
"type": "image/jpeg",
55+
"roles": [
56+
"thumbnail"
57+
]
58+
}
59+
},
60+
"renders": {
61+
"dashboard": {
62+
"resampling": "nearest",
63+
"bidx": [
64+
1
65+
],
66+
"colormap_name": "nlcd",
67+
"nodata": 0,
68+
"assets": [
69+
"cog_default"
70+
],
71+
"title": "VEDA Dashboard Render Parameters"
72+
}
73+
},
74+
"dashboard:is_periodic": false,
75+
"dashboard:time_density": "year"
76+
}

‎ingestion-data/staging/dataset-config/test.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
2-
"collection": "campfire-nlcd-TEST-HASH",
3-
"title": "Camp Fire Domain: Land Cover TEST HASH",
2+
"collection": "campfire-nlcd-TEST-CHANGE",
3+
"title": "Camp Fire Domain: Land Cover TEST CHANGE",
44
"description": "NLCD-derived land use-land cover for 2016 over the Camp Fire burn scar domain. Displays colors 0-255 from a lookup table for land cover type (ex: reds = urbanized).",
55
"type": "Collection",
66
"spatial_extent": {

0 commit comments

Comments
 (0)