Skip to content

Commit ad78fd4

Browse files
linglplinglp
authored
Merge pull request #1032 from Sage-Bionetworks/develop
Release 22.11.3
2 parents 322cf42 + 1887b0a commit ad78fd4

File tree

14 files changed

+604
-290
lines changed

14 files changed

+604
-290
lines changed

‎.github/workflows/publish.yml‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,9 +92,29 @@ jobs:
9292
# publish to pypi
9393
#----------------------------------------------
9494
- name: Publish package to Pypi
95+
id: publish-to-pypi
9596
if: steps.check-tag.outputs.match == 'true'
9697
env:
9798
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
9899
PYPI_USERNAME: __token__
99100
run: |
100-
poetry publish --build --username $PYPI_USERNAME --password $PYPI_TOKEN
101+
poetry publish --build --username $PYPI_USERNAME --password $PYPI_TOKEN
102+
103+
#----------------------------------------------
104+
# post a message to slack
105+
#----------------------------------------------
106+
107+
- name: Post to a Slack channel
108+
if: steps.publish-to-pypi.outcome == 'success'
109+
id: slack
110+
uses: slackapi/slack-github-action@v1.23.0
111+
with:
112+
# Slack channel id, channel name, or user id to post message.
113+
# See also: https://api.slack.com/methods/chat.postMessage#channels
114+
# You can pass in multiple channels to post to by providing a comma-delimited list of channel IDs.
115+
# ibc-fair-data channel and data-curator-schematic channel
116+
channel-id: 'C01HSSMPQBG,C01ANC02U59'
117+
# For posting a simple plain text message
118+
slack-message: "Schematic has just been released. Check out new version: ${{ github.ref_name }}"
119+
env:
120+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

‎api/openapi/api.yaml‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -747,8 +747,39 @@ paths:
747747
description: Check schematic log.
748748
tags:
749749
- Schema Operation
750-
751-
750+
/schemas/is_node_required:
751+
get:
752+
summary: Check if a node is required or not
753+
description: Check if a node is required or not
754+
operationId: api.routes.get_if_node_required
755+
parameters:
756+
- in: query
757+
name: schema_url
758+
schema:
759+
type: string
760+
description: Data Model URL
761+
example: >-
762+
https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.model.jsonld
763+
required: true
764+
- in: query
765+
name: node_display_name
766+
schema:
767+
type: string
768+
nullable: false
769+
description: Display label of a node
770+
example: FamilyHistory
771+
required: true
772+
responses:
773+
"200":
774+
description: return a boolean
775+
"500":
776+
description: Check schematic log.
777+
tags:
778+
- Schema Operation
779+
780+
781+
782+
752783
/explorer/get_node_dependencies:
753784
get:
754785
summary: Get the immediate dependencies that are related to a given source node

‎api/routes.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -673,3 +673,21 @@ def get_node_range(
673673
gen = SchemaGenerator(path_to_json_ld=schema_url)
674674
node_range = gen.get_node_range(node_label, return_display_names)
675675
return node_range
676+
677+
def get_if_node_required(schema_url: str, node_display_name: str) -> bool:
678+
"""Check if the node is required
679+
680+
Args:
681+
schema_url (str): Data Model URL
682+
node_display_name (str): display name
683+
684+
Returns:
685+
True: If the given node is a "required" node.
686+
False: If the given node is not a "required" (i.e., an "optional") node.
687+
"""
688+
gen = SchemaGenerator(path_to_json_ld=schema_url)
689+
is_required = gen.is_node_required(node_display_name)
690+
691+
return is_required
692+
693+

‎schematic/models/GE_Helpers.py‎

Lines changed: 32 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from great_expectations.data_context.types.resource_identifiers import ExpectationSuiteIdentifier
2424

2525
from schematic.models.validate_attribute import GenerateError
26+
from schematic.schemas.generator import SchemaGenerator
2627
from schematic.utils.validate_utils import rule_in_rule_list
2728

2829
logger = logging.getLogger(__name__)
@@ -159,7 +160,7 @@ def build_expectation_suite(self,):
159160
if validation_rules:
160161
#iterate through all validation rules for an attribute
161162
for rule in validation_rules:
162-
163+
base_rule = rule.split(" ")[0]
163164

164165
#check if rule has an implemented expectation
165166
if rule_in_rule_list(rule,self.unimplemented_expectations):
@@ -169,8 +170,9 @@ def build_expectation_suite(self,):
169170
args["column"] = col
170171
args["result_format"] = "COMPLETE"
171172

173+
172174
#Validate num
173-
if rule=='num':
175+
if base_rule=='num':
174176
args["mostly"]=1.0
175177
args["type_list"]=['int','int64', 'float', 'float64']
176178
meta={
@@ -182,7 +184,7 @@ def build_expectation_suite(self,):
182184
}
183185

184186
#Validate float
185-
elif rule=='float':
187+
elif base_rule=='float':
186188
args["mostly"]=1.0
187189
args["type_list"]=['float', 'float64']
188190
meta={
@@ -194,7 +196,7 @@ def build_expectation_suite(self,):
194196
}
195197

196198
#Validate int
197-
elif rule=='int':
199+
elif base_rule=='int':
198200
args["mostly"]=1.0
199201
args["type_list"]=['int','int64']
200202
meta={
@@ -206,7 +208,7 @@ def build_expectation_suite(self,):
206208
}
207209

208210
#Validate string
209-
elif rule=='str':
211+
elif base_rule=='str':
210212
args["mostly"]=1.0
211213
args["type_"]='str'
212214
meta={
@@ -217,7 +219,7 @@ def build_expectation_suite(self,):
217219
"validation_rule": rule
218220
}
219221

220-
elif rule.startswith("recommended"):
222+
elif base_rule==("recommended"):
221223
args["mostly"]=0.0000000001
222224
args["regex_list"]=['^$']
223225
meta={
@@ -228,7 +230,7 @@ def build_expectation_suite(self,):
228230
"validation_rule": rule
229231
}
230232

231-
elif rule.startswith("protectAges"):
233+
elif base_rule==("protectAges"):
232234
#Function to convert to different age limit formats
233235
min_age, max_age = self.get_age_limits()
234236

@@ -243,7 +245,7 @@ def build_expectation_suite(self,):
243245
"validation_rule": rule
244246
}
245247

246-
elif rule.startswith("unique"):
248+
elif base_rule==("unique"):
247249
args["mostly"]=1.0
248250
meta={
249251
"notes": {
@@ -253,7 +255,7 @@ def build_expectation_suite(self,):
253255
"validation_rule": rule
254256
}
255257

256-
elif rule.startswith("inRange"):
258+
elif base_rule==("inRange"):
257259
args["mostly"]=1.0
258260
args["min_value"]=float(rule.split(" ")[1])
259261
args["max_value"]=float(rule.split(" ")[2])
@@ -350,7 +352,8 @@ def generate_errors(
350352
validation_results: Dict,
351353
validation_types: Dict,
352354
errors: List,
353-
warnings: List
355+
warnings: List,
356+
sg: SchemaGenerator,
354357
):
355358
"""
356359
Purpose:
@@ -407,45 +410,50 @@ def generate_errors(
407410
#call functions to generate error messages and add to error list
408411
if validation_types[rule.split(" ")[0]]['type']=='type_validation':
409412
for row, value in zip(indices,values):
410-
errors.append(
411-
GenerateError.generate_type_error(
413+
vr_errors, vr_warnings = GenerateError.generate_type_error(
412414
val_rule = rule,
413415
row_num = row+2,
414416
attribute_name = errColumn,
415417
invalid_entry = value,
418+
sg = sg,
416419
)
417-
)
420+
if vr_errors:
421+
errors.append(vr_errors)
422+
if vr_warnings:
423+
warnings.append(vr_warnings)
418424
elif validation_types[rule.split(" ")[0]]['type']=='regex_validation':
419425
expression=result_dict['expectation_config']['kwargs']['regex']
420-
421426
for row, value in zip(indices,values):
422-
errors.append(
423-
GenerateError.generate_regex_error(
427+
vr_errors, vr_warnings = GenerateError.generate_regex_error(
424428
val_rule= rule,
425429
reg_expression = expression,
426430
row_num = row+2,
427431
module_to_call = 'match',
428432
attribute_name = errColumn,
429433
invalid_entry = value,
434+
sg = sg,
430435
)
431-
)
436+
if vr_errors:
437+
errors.append(vr_errors)
438+
if vr_warnings:
439+
warnings.append(vr_warnings)
432440
elif validation_types[rule.split(" ")[0]]['type']=='content_validation':
433-
content_errors, content_warnings = GenerateError.generate_content_error(
441+
vr_errors, vr_warnings = GenerateError.generate_content_error(
434442
val_rule = rule,
435443
attribute_name = errColumn,
436444
row_num = list(np.array(indices)+2),
437445
error_val = values,
438446
sg = self.sg
439447
)
440-
if content_errors:
441-
errors.append(content_errors)
448+
if vr_errors:
449+
errors.append(vr_errors)
442450
if rule.startswith('protectAges'):
443-
self.censor_ages(content_errors,errColumn)
451+
self.censor_ages(vr_errors,errColumn)
444452
pass
445-
elif content_warnings:
446-
warnings.append(content_warnings)
453+
if vr_warnings:
454+
warnings.append(vr_warnings)
447455
if rule.startswith('protectAges'):
448-
self.censor_ages(content_warnings,errColumn)
456+
self.censor_ages(vr_warnings,errColumn)
449457
pass
450458

451459
return errors, warnings

0 commit comments

Comments
 (0)