Skip to content

Commit 9d88c77

Browse files
esggesgg
committed
Code to process datasets using f-uji
1 parent 57583e6 commit 9d88c77

1 file changed

Lines changed: 141 additions & 0 deletions

File tree

‎code/fair_assessment/algorithms/FAIROS_DATASET_FUJI.py‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import logging
2+
import traceback
3+
from rocrate.rocrate import ROCrate
4+
import validators
5+
from fuji_wrapper.fujiwrapper import FujiWrapper
6+
from rdflib import Graph, Namespace, URIRef
7+
import json
8+
from datetime import datetime, timezone
9+
from pathlib import Path
10+
from rdflib.namespace import DCTERMS
11+
12+
logger = logging.getLogger(__name__)
13+
14+
class FAIROS_DATASET_FUJI:
15+
16+
mappings={
17+
"Identifier is resolvable and follows a defined unique identifier syntax (IRI, URL)":"https://w3id.org/FAIROS/test/FUJI-F1-01",
18+
"Identifier follows a defined persistent identifier syntax":"https://w3id.org/FAIROS/test/FUJI-F1-02",
19+
"Metadata includes descriptive core elements (creator, title, data identifier, publisher, publication date, summary and keywords) to support data findability.":"https://w3id.org/FAIROS/test/FUJI-F2-02",
20+
"Metadata has been made available via common web methods":"https://w3id.org/FAIROS/test/FUJI-F4-01",
21+
"Core data citation metadata is available":"https://w3id.org/FAIROS/test/FUJI-F2-01",
22+
"Core descriptive metadata is available":"https://w3id.org/FAIROS/test/FUJI-F2-02",
23+
"Metadata includes the identifier of the data it describes.":"https://w3id.org/FAIROS/test/FUJI-F3-01",
24+
"Metadata contains a PID or URL which indicates the location of the downloadable data content":"https://w3id.org/FAIROS/test/FUJI-F3-01",
25+
"Metadata is given in a way major search engines can ingest it for their catalogues (embedded JSON-LD, Dublin Core or RDFa)":"https://w3id.org/FAIROS/test/FAIR-I1-01",
26+
"Information about access restrictions or rights can be identified in metadata":"https://w3id.org/FAIROS/test/FUJI-A1-01",
27+
"Metadata is accessible through a standardized communication protocol.":"https://w3id.org/FAIROS/test/FUJI-A1-02",
28+
"Data access information is indicated by (not machine readable) standard terms":"https://w3id.org/FAIROS/test/FUJI-A1-03",
29+
"Parsable, structured metadata (JSON-LD, RDFa) is embedded in the landing page XHTML/HTML code":"https://w3id.org/FAIROS/test/FUJI-I1-01",
30+
"Parsable, graph data (RDF, JSON-LD) is accessible through content negotiation, typed links or sparql endpoint":"https://w3id.org/FAIROS/test/FUJI-I1-02",
31+
"Metadata uses semantic resources":"https://w3id.org/FAIROS/test/FUJI-I1-02",
32+
"Vocabulary namespace URIs can be identified in metadata":"https://w3id.org/FAIROS/test/FUJI-I2-01",
33+
"Metadata includes links between the data and its related entities.":"https://w3id.org/FAIROS/test/FUJI-I3-01",
34+
"Related resources are indicated by machine readable links or identifiers":"https://w3id.org/FAIROS/test/FUJI-I3-02",
35+
"Minimal information about available data content is given in metadata":"https://w3id.org/FAIROS/test/FUJI-R1-01",
36+
"Information about data content (e.g. links) is given in metadata":"https://w3id.org/FAIROS/test/FUJI-R1-02",
37+
"Data content matches measured variables or observation types specified in metadata":"https://w3id.org/FAIROS/test/FUJI-R1-03",
38+
"Licence information is given in an appropriate metadata element":"https://w3id.org/FAIROS/test/FUJI-R1.1-01",
39+
"Metadata contains elements which hold provenance information and can be mapped to PROV":"https://w3id.org/FAIROS/test/FUJI-R1.2-01",
40+
"Metadata contains provenance information using formal provenance ontologies (PROV-O)":"https://w3id.org/FAIROS/test/FUJI-R1.2-02",
41+
"Multidisciplinary but community endorsed metadata (RDA Metadata Standards Catalog, fairsharing) standard is listed in the re3data record or detected by namespace":"https://w3id.org/FAIROS/test/FUJI-R1.3-01",
42+
"The format of a data file given in the metadata is listed in the long term file formats, open file formats or scientific file formats controlled list":"https://w3id.org/FAIROS/test/FUJI-R1.3-02"
43+
}
44+
45+
def __init__(self):
46+
47+
FTR = Namespace("https://w3id.org/ftr#")
48+
49+
logger.info("Load Algorithm description")
50+
self.tests_graph = Graph()
51+
self.tests_graph.parse("fair_assessment/algorithms/descriptions/FAIROS_DATASET_FUJI.ttl", format="turtle")
52+
53+
tests_directories = ["C:\\Users\\egonzalez\\projects\\FAIR-Research-Object\\docs\\catalog\\tests"]
54+
55+
# Recursively find and load all .ttl files of tests
56+
logger.info("Loading test description files")
57+
for directory in tests_directories:
58+
for ttl_file in Path(directory).rglob("*.ttl"):
59+
logger.info(f"Loading {ttl_file}...")
60+
self.tests_graph.parse(ttl_file, format="turtle")
61+
62+
print(f"Loaded {len(self.tests_graph)} triples from all TTL files.")
63+
64+
def execute_algorithm(self, rocrate_dataset, ticket):
65+
66+
logger.info("Invoking F-UJI")
67+
68+
if validators.url(rocrate_dataset["@id"]):
69+
fuji = FujiWrapper(rocrate_dataset["@id"])
70+
71+
# Current UTC time
72+
now = datetime.now(timezone.utc)
73+
formatted = now.strftime("%a %b %d %H:%M:%S UTC %Y")
74+
75+
doc = {
76+
"@context": "https://w3id.org/ftr/context",
77+
"@id": f"urn:fairos:{ticket}",
78+
"@type": "https://w3id.org/ftr#TestResultSet",
79+
"description": (
80+
"Set of test results that includes all tests included "
81+
"in the Algorithm FAIROS_DATASET"
82+
),
83+
"identifier": {
84+
"@id": f"urn:fairos:{ticket}"
85+
},
86+
"assessmentTarget": {
87+
"@id": f"{rocrate_dataset['@id']}"
88+
},
89+
"license": {
90+
"@id": "http://creativecommons.org/licenses/by/4.0/"
91+
},
92+
"title": "Results from running FAIROS DATASETS!",
93+
"generatedAtTime": {
94+
"@type": "http://www.w3.org/2001/XMLSchema#date",
95+
"@value": f"{formatted}"
96+
},
97+
"hadMember":[]
98+
}
99+
100+
#Get results from f-uji
101+
results_json = fuji.get_checks()
102+
103+
#Explore results, divided in metrics
104+
for item in results_json:
105+
explanations = []
106+
for src in item.get("sources", []):
107+
explanations.extend(src.get("explanation", []))
108+
#Each explanation is a test results
109+
for explanation in explanations:
110+
if ":" in explanation:
111+
#The result is an string with PASS: test_name or FAIL: test_name
112+
value, test_text = explanation.split(':', 1)
113+
test_text = test_text.strip()
114+
115+
test_uri = self.mappings.get(test_text)
116+
117+
if test_uri:
118+
description = self.tests_graph.value(subject=URIRef(test_uri), predicate=DCTERMS.description)
119+
title = self.tests_graph.value(subject=URIRef(test_uri), predicate=DCTERMS.title)
120+
121+
test = {
122+
"@id": f"urn:fairos:{ticket}",
123+
"@type": "https://w3id.org/ftr#TestResult",
124+
"description": f"{description}",
125+
"identifier": {
126+
"@id": f"urn:fairos:{ticket}"
127+
},
128+
"license": {
129+
"@id": "http://creativecommons.org/licenses/by/4.0/"
130+
},
131+
"title": f"{title}",
132+
"value": f"{value}",
133+
"completion": {
134+
"@value": 100
135+
},
136+
"log": "N/A"
137+
}
138+
139+
doc["hadMember"].append(test)
140+
else:
141+
logger.error("Invalid url")

0 commit comments

Comments
 (0)