Skip to content

Commit 17d3c84

Browse files
xinli-caixinli-cai
committed
June 2024 release update for adding sourceSystemName used for EO filters
1 parent 1413ff8 commit 17d3c84

3 files changed

Lines changed: 145 additions & 38 deletions

File tree

‎stac-to-geocore/app.py‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,23 +11,25 @@
1111
from s3_operations import *
1212
from stac_to_geocore import *
1313

14-
"""
14+
1515
# environment variables for lambda
1616
geocore_template_bucket_name = os.environ['GEOCORE_TEMPLATE_BUCKET_NAME']
1717
geocore_template_name = os.environ['GEOCORE_TEMPLATE_NAME']
1818
geocore_to_parquet_bucket_name = os.environ['GEOCORE_TO_PARQUET_BUCKET_NAME']
1919
api_root = os.environ['STAC_API_ROOT']
2020
root_name = os.environ['ROOT_NAME']
2121
source = os.environ['SOURCE']
22+
sourceSystemName = os.environ['SOURCESYSTEMNAME']
23+
2224

23-
"""
24-
#dev setting -- comment out for release
25-
geocore_template_bucket_name = 'webpresence-geocore-template-dev'
26-
geocore_template_name = 'geocore-format-null-template.json'
27-
geocore_to_parquet_bucket_name = "webpresence-geocore-json-to-geojson-dev" #s3 for geocore to parquet translation
28-
api_root = 'https://datacube.services.geo.ca/api'
29-
root_name = "CCMEO Datacube/ CCCOT Cube de données" #must provide en and fr
30-
source='ccmeo'
25+
# #dev setting -- comment out for release
26+
# geocore_template_bucket_name = 'webpresence-geocore-template-dev'
27+
# geocore_template_name = 'geocore-format-null-template.json'
28+
# geocore_to_parquet_bucket_name = "webpresence-geocore-json-to-geojson-dev" #s3 for geocore to parquet translation
29+
# api_root = 'https://datacube.services.geo.ca/api'
30+
# root_name = "CCMEO Datacube/ CCCOT Cube de données" #must provide en and fr
31+
# source='ccmeo'
32+
# sourceSystemName = 'ccmeo-datacube'
3133

3234

3335

@@ -57,6 +59,8 @@ def lambda_handler(event, context):
5759
# Start the harvest and translation process if connection is okay
5860
if response_root.status_code == 200:
5961
#Delete previous harvest included in lastRun.txt
62+
print(geocore_to_parquet_bucket_name)
63+
print(geocore_template_bucket_name)
6064
e = delete_stac_s3(bucket_geojson=geocore_to_parquet_bucket_name, bucket_template=geocore_template_bucket_name)
6165
if e != None:
6266
error_msg += e
@@ -97,7 +101,8 @@ def lambda_handler(event, context):
97101
'spatialRepresentation': spatialRepresentation,
98102
'contact': contact,
99103
'type_data': type_data,
100-
'topicCategory': topicCategory
104+
'topicCategory': topicCategory,
105+
'sourceSystemName': sourceSystemName
101106
}
102107
#print(f"This is the geocore_features_dict when it is in the params before root_to_features_properties\n {params['geocore_features_dict']}")
103108
root_properties_dict = root_to_features_properties(params, geocore_features_dict)

‎stac-to-geocore/stac_to_geocore.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,7 @@ def root_to_features_properties(params, geocore_features_dict):
387387
contact = params['contact']
388388
type_data = params['type_data']
389389
topicCategory = params['topicCategory']
390-
390+
sourceSystemName = params['sourceSystemName']
391391

392392
properties_dict = geocore_features_dict['properties']
393393
root_name_en,root_name_fr = root_name.split('/')
@@ -427,7 +427,9 @@ def root_to_features_properties(params, geocore_features_dict):
427427
'contact': contact,
428428
'options': options_list,
429429
'useLimits': {'en': useLimits_en, 'fr': useLimits_fr},
430-
'temporalExtent': {'end': 'Present', 'begin': '0001-01-01'}
430+
'temporalExtent': {'end': 'Present', 'begin': '0001-01-01'},
431+
"sourceSystemName": sourceSystemName,
432+
431433
})
432434
#parentIdentifier: None for STAC catalog
433435
#date: None for STAC collection
@@ -452,6 +454,7 @@ def coll_to_features_properties(params, coll_dict,geocore_features_dict):
452454
contact = params['contact']
453455
type_data = params['type_data']
454456
topicCategory = params['topicCategory']
457+
sourceSystemName = params["sourceSystemName"]
455458

456459
properties_dict = geocore_features_dict['properties']
457460

@@ -500,6 +503,7 @@ def coll_to_features_properties(params, coll_dict,geocore_features_dict):
500503
'description': {'en': description_en_str, 'fr': description_fr_str},
501504
'keywords': {'en': keywords_en_str, 'fr': keywords_fr_str},
502505
"geometry": geometry_str,
506+
"sourceSystemName": sourceSystemName,
503507

504508
})
505509

@@ -575,6 +579,7 @@ def item_to_features_properties(params, geocore_features_dict, item_dict, coll_i
575579
contact = params['contact']
576580
type_data = params['type_data']
577581
topicCategory = params['topicCategory']
582+
sourceSystemName = params['sourceSystemName']
578583

579584
properties_dict = geocore_features_dict['properties']
580585
# Get item level lelments
@@ -663,6 +668,7 @@ def item_to_features_properties(params, geocore_features_dict, item_dict, coll_i
663668
'description': {'en': description_en_str, 'fr': description_fr_str},
664669
'keywords': {'en': keywords_en_str, 'fr': keywords_fr_str},
665670
"geometry": geometry_str,
671+
"sourceSystemName": sourceSystemName,
666672

667673
})
668674

@@ -671,7 +677,7 @@ def item_to_features_properties(params, geocore_features_dict, item_dict, coll_i
671677
#skipped metadataStandard, metadataStandardVersion, metadataStandardVersion, graphicOverview, distributionFormat_name, distributionFormat_format
672678
#skipped: accessConstraints, otherConstraints, dateStamp, dataSetURI, locale,language
673679
#skipped: characterSet, environmentDescription,supplementalInformation
674-
#skipped: credits, cited, distributor,sourceSystemName
680+
#skipped: credits, cited, distributor
675681
# options
676682
return (properties_dict)
677683

‎template.yaml‎

Lines changed: 121 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,127 @@
11
AWSTemplateFormatVersion: '2010-09-09'
2-
Transform: AWS::Serverless-2016-10-31
3-
Description: >
4-
python3.9
5-
6-
Sample SAM Template for STAC_to_geoCore
7-
Resources:
8-
STACtoGeoCoreNewFunction:
9-
Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction
2+
Transform: 'AWS::Serverless-2016-10-31'
3+
Description: Deploys STAC datacube harvester solution + STAC to geocore translation
4+
5+
Parameters:
6+
Environment:
7+
Type: AWS::SSM::Parameter::Value<String>
8+
Default: /webpresence/environment
9+
Description: SSM parameter name for environment
10+
DeploymentBucket:
11+
Type: AWS::SSM::Parameter::Value<String>
12+
Default: /webpresence/deployment-bucket
13+
Description: S3 bucket where all deployment files are stored
14+
15+
16+
Conditions:
17+
IsProd: !Equals [prod, !Ref Environment]
18+
IsStage: !Equals [stage, !Ref Environment]
19+
IsDev: !Equals [dev, !Ref Environment]
20+
21+
Resources:
22+
GeocoreFormatTemplateBucket:
23+
Type: AWS::S3::Bucket
1024
Properties:
25+
BucketName: !Sub 'webpresence-geocore-template-${Environment}'
26+
BucketEncryption:
27+
ServerSideEncryptionConfiguration:
28+
- ServerSideEncryptionByDefault:
29+
SSEAlgorithm: AES256
30+
PublicAccessBlockConfiguration:
31+
BlockPublicAcls: True
32+
BlockPublicPolicy: True
33+
IgnorePublicAcls: True
34+
RestrictPublicBuckets: True
35+
AccessControl: Private
36+
37+
GeocoreStacHarvestAndTransformFunction:
38+
Type: AWS::Serverless::Function
39+
Properties:
40+
Runtime: python3.9
41+
Role: !GetAtt LambdaExecutionRole.Arn
42+
CodeUri:
43+
Bucket: !Ref DeploymentBucket
44+
Key:
45+
Fn::If:
46+
- IsProd
47+
- cloudformation-templates/lambda/stac-to-geocore/stac-to-geocore-20240628-1430.zip
48+
- Fn::If:
49+
- IsStage
50+
- cloudformation-templates/lambda/stac-to-geocore/stac-to-geocore-20240628-1430.zip
51+
- cloudformation-templates/lambda/stac-to-geocore/stac-to-geocore-20240628-1430.zip
52+
MemorySize: 4096
1153
Handler: app.lambda_handler
12-
CodeUri: stac-to-geocore #required if PackageType is Zip. The path to app.py and requirement.txt
13-
Description: ''
14-
MemorySize: 1024
1554
Timeout: 900
16-
Runtime: python3.9
17-
Architectures:
18-
- x86_64
19-
Role: >-
20-
arn:aws:iam::006288227511:role/hnap-json-harvester-LambdaExecutionRole-YGC4NRBOO685
2155
Environment:
2256
Variables:
23-
GEOCORE_TEMPLATE_BUCKET_NAME: webpresence-geocore-template-dev
24-
STAC_BUCKET_NAME: webpresence-stac-json-dev
25-
STAC_GEOCORE_BUCKET_NAME: webpresence-geocore-json-to-geojson-dev
26-
Events:
27-
STACtoGeoCore:
28-
Type: Api # More info about API Event Source: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#api
29-
Properties:
30-
Path: /stac_to_geocore
31-
Method: get
57+
GEOCORE_TEMPLATE_BUCKET_NAME: !Ref GeocoreFormatTemplateBucket
58+
GEOCORE_TEMPLATE_NAME: 'geocore-format-null-template.json'
59+
GEOCORE_TO_PARQUET_BUCKET_NAME: !Sub 'webpresence-geocore-json-to-geojson-${Environment}'
60+
STAC_API_ROOT: 'https://datacube.services.geo.ca/api'
61+
ROOT_NAME: 'CCMEO Datacube API / CCCOT Cube de données API'
62+
SOURCE: 'ccmeo'
63+
SOURCESYSTEMNAME: 'ccmeo-datacube'
64+
Layers:
65+
- arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:8
66+
67+
StacHarvesterRule:
68+
Type: AWS::Events::Rule
69+
Properties:
70+
Name: !Sub 'stac-harvester-1day-${Environment}'
71+
Description: Stac Harvester on a 1 day interval
72+
State: ENABLED
73+
ScheduleExpression: 'rate(1 day)'
74+
Targets:
75+
-
76+
Arn:
77+
Fn::GetAtt:
78+
- GeocoreStacHarvestAndTransformFunction
79+
- Arn
80+
Id: !Ref GeocoreStacHarvestAndTransformFunction
81+
82+
83+
PermissionForEventsToInvokeStacHarvesterLambda:
84+
Type: AWS::Lambda::Permission
85+
Properties:
86+
FunctionName: !Ref GeocoreStacHarvestAndTransformFunction
87+
Action: 'lambda:InvokeFunction'
88+
Principal: 'events.amazonaws.com'
89+
SourceArn: !GetAtt
90+
- StacHarvesterRule
91+
- Arn
92+
93+
LambdaExecutionRole:
94+
Type: AWS::IAM::Role
95+
Properties:
96+
AssumeRolePolicyDocument:
97+
Version: '2012-10-17'
98+
Statement:
99+
- Effect: Allow
100+
Principal:
101+
Service:
102+
- lambda.amazonaws.com
103+
Action:
104+
- sts:AssumeRole
105+
ManagedPolicyArns:
106+
- arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
107+
- arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess
108+
Policies:
109+
- PolicyName: 'policy'
110+
PolicyDocument:
111+
Version: '2012-10-17'
112+
Statement:
113+
- Effect: 'Allow'
114+
Action:
115+
- 's3:*'
116+
Resource:
117+
- !Sub arn:aws:s3:::webpresence-geocore-template-${Environment}/*
118+
- !Sub arn:aws:s3:::webpresence-geocore-template-${Environment}
119+
- !Sub arn:aws:s3:::webpresence-geocore-json-to-geojson-${Environment}/*
120+
- !Sub arn:aws:s3:::webpresence-geocore-json-to-geojson-${Environment}
121+
122+
LogGroup:
123+
Type: Custom::LogGroup
124+
Properties:
125+
ServiceToken: !ImportValue LogGroupHelperLambdaArn
126+
LogGroupName: !Sub '/${Environment}/webpresence/stac_harvester'
127+
RetentionInDays: 3653

0 commit comments

Comments
 (0)