Skip to content

Commit 7873229

Browse files
benjaminaaronbenjaminaaron
committed
Change approach: retrieve metadata separately
1 parent 8773f82 commit 7873229

File tree

4 files changed

+80
-73
lines changed

4 files changed

+80
-73
lines changed

‎dev/dev.js

+14-9
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,16 @@ import path from "path"
33
import { fileURLToPath } from "url"
44
import fs, { promises as fsPromise } from "fs"
55
import { validateAll, validateOne, validateUserProfile } from "../src/index.js"
6-
import { extractRequirementProfilesMetadata, runSparqlConstructQueryOnRdfString, runSparqlSelectQueryOnRdfString } from "../src/utils.js"
6+
import {
7+
extractDatafieldsMetadata,
8+
extractRequirementProfilesMetadata,
9+
runSparqlConstructQueryOnRdfString,
10+
runSparqlSelectQueryOnRdfString
11+
} from "../src/utils.js"
712

813
const DB_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "requirement-profiles")
914
const SHACL_DIR = `${DB_DIR}/shacl`
10-
const USER_PROFILE = `${DB_DIR}/user-profile-examples/kinderzuschlag-user-profile.ttl`
15+
const USER_PROFILE = `${DB_DIR}/user-profile-examples/empty-user-profile.ttl`
1116
const DATAFIELDS = `${DB_DIR}/datafields.ttl`
1217
const MATERIALIZATION = `${DB_DIR}/materialization.ttl`
1318

@@ -100,13 +105,13 @@ async function devValidateUserProfile() {
100105
console.log(conforms)
101106
}
102107

103-
async function devExtractRequirementProfileMedatada() {
104-
let shaclFiles = await fsPromise.readdir(SHACL_DIR)
105-
let shaclFileContents = []
106-
for (let file of shaclFiles) {
107-
shaclFileContents.push(await fsPromise.readFile(`${SHACL_DIR}/${file}`, "utf8"))
108+
async function devExtractMedatada() {
109+
let rpStrings = []
110+
for (let file of await fsPromise.readdir(SHACL_DIR)) {
111+
rpStrings.push(await fsPromise.readFile(`${SHACL_DIR}/${file}`, "utf8"))
108112
}
109-
console.log(await extractRequirementProfilesMetadata(shaclFileContents))
113+
console.log("Requirement profiles metadata:", await extractRequirementProfilesMetadata(rpStrings))
114+
console.log("Datafields metadata:", await extractDatafieldsMetadata(await fsPromise.readFile(DATAFIELDS, "utf8")))
110115
}
111116

112117
// devRunSparqlSelectQueryOnRdfString()
@@ -115,4 +120,4 @@ devValidateAll()
115120
// devValidateOne()
116121
// devValidateOneStrings()
117122
// devValidateUserProfile()
118-
// devExtractRequirementProfileMedatada()
123+
// devExtractMedatada()

‎global.js

+4-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ import {
88
import {
99
rdfStringsToStore,
1010
runSparqlSelectQueryOnStore,
11-
extractRequirementProfilesMetadata
11+
extractRequirementProfilesMetadata,
12+
extractDatafieldsMetadata
1213
} from "./src/utils.js";
1314

1415
window.MatchingEngine = {
@@ -18,5 +19,6 @@ window.MatchingEngine = {
1819
validateAll,
1920
rdfStringsToStore,
2021
runSparqlSelectQueryOnStore,
21-
extractRequirementProfilesMetadata
22+
extractRequirementProfilesMetadata,
23+
extractDatafieldsMetadata
2224
}

‎src/index.js

+14-44
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import {
2-
addRdfStringToStore,
3-
extractRequirementProfilesMetadataFromStore,
2+
addRdfStringToStore, extractRpUriFromRpString,
43
printDatasetAsTurtle,
54
printStoreAsTurtle,
65
runSparqlAskQueryOnStore,
@@ -40,31 +39,24 @@ export async function validateUserProfile(userProfile, datafieldsStr, debug = fa
4039
export async function validateAll(userProfileStr, requirementProfiles, datafieldsStr, materializationStr, debug = false) {
4140
let map = {
4241
reports: [],
43-
missingUserInputsAggregated: {},
44-
metadata: {}
42+
missingUserInputsAggregated: {}
4543
}
4644
for (let [filename, reqProfileStr] of Object.entries(requirementProfiles)) {
45+
let rpUri = await extractRpUriFromRpString(reqProfileStr)
4746
let report = await validateOne(userProfileStr, reqProfileStr, datafieldsStr, materializationStr, debug)
47+
report.rpUri = rpUri
4848
map.reports.push(report)
49-
map.metadata = { ...map.metadata, ...report.metadata }
50-
let rpId = Object.keys(report.metadata)[0]
51-
report.metadata = {
52-
...report.metadata[rpId],
53-
id: rpId,
54-
filename: filename
55-
}
5649
for (let userInput of report.missingUserInput) {
57-
let key = userInput.subject + "_" + userInput.predicate
50+
let key = userInput.subject + "_" + userInput.dfUri
5851
if (!map.missingUserInputsAggregated[key]) {
5952
map.missingUserInputsAggregated[key] = {
6053
subject: userInput.subject,
61-
predicate: userInput.predicate,
62-
label: userInput.label,
54+
dfUri: userInput.dfUri,
6355
usedIn: []
6456
}
6557
}
6658
map.missingUserInputsAggregated[key].usedIn.push({
67-
id: rpId,
59+
rpUri: rpUri,
6860
optional: userInput.optional,
6961
isLastMissingUserInput: report.missingUserInput.length === 1
7062
})
@@ -82,9 +74,6 @@ export async function validateOne(userProfile, requirementProfile, datafieldsStr
8274
await addRdfStringToStore(materializationStr, store)
8375
await addRdfStringToStore(datafieldsStr, store)
8476

85-
// ----- extract metadata from the requirement profile -----`
86-
let rpMetadata = await extractRequirementProfilesMetadataFromStore(store)
87-
8877
// ----- first validation to identify missing data points -----
8978
let firstReport = await runValidationOnStore(store)
9079
if (debug) {
@@ -100,8 +89,7 @@ export async function validateOne(userProfile, requirementProfile, datafieldsStr
10089
result: ValidationResult.INELIGIBLE,
10190
violations: violations,
10291
missingUserInput: [],
103-
inMemoryMaterializedTriples: [],
104-
metadata: rpMetadata
92+
inMemoryMaterializedTriples: []
10593
}
10694
}
10795

@@ -114,7 +102,7 @@ export async function validateOne(userProfile, requirementProfile, datafieldsStr
114102
let message = result.message[0].value // can the arrays be bigger than 1?
115103
missingList.push({
116104
subject: fromSubject,
117-
predicate: missingPredicate,
105+
dfUri: missingPredicate, // predicate
118106
optional: message.toLowerCase().includes("[optional]") // a better way to check for this?
119107
})
120108
}
@@ -130,7 +118,7 @@ export async function validateOne(userProfile, requirementProfile, datafieldsStr
130118
PREFIX ff: <https://foerderfunke.org/default#>
131119
SELECT * WHERE {
132120
?rule ff:output ?output .
133-
FILTER(?output = <${missing.predicate}>) .
121+
FILTER(?output = <${missing.dfUri}>) .
134122
?rule ff:sparqlConstructQuery ?query .
135123
OPTIONAL { ?rule ff:input ?input . }
136124
}
@@ -155,7 +143,7 @@ export async function validateOne(userProfile, requirementProfile, datafieldsStr
155143
let existingMainPersonPredicates = result.map(n => n.predicate)
156144

157145
for (let missing of missingList) {
158-
let matchingRule = materializableDataPoints.find(n => n.output === missing.predicate)
146+
let matchingRule = materializableDataPoints.find(n => n.output === missing.dfUri)
159147
let otherRuleWithThatInputAsOutput = undefined
160148
if (matchingRule && matchingRule.input) {
161149
otherRuleWithThatInputAsOutput = materializableDataPoints.find(n => n.output === matchingRule.input)
@@ -166,22 +154,6 @@ export async function validateOne(userProfile, requirementProfile, datafieldsStr
166154
askUserForDataPoints.push(missing)
167155
}
168156

169-
// ----- enrich the ones we'll ask for with labels -----
170-
// should we send more along than then the label?
171-
for (let dataPoint of askUserForDataPoints) {
172-
let query = `
173-
PREFIX ff: <https://foerderfunke.org/default#>
174-
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
175-
SELECT * WHERE {
176-
?predicate a ff:DataField .
177-
FILTER(?predicate = <${dataPoint.predicate}>) .
178-
?predicate rdfs:label ?label .
179-
}
180-
`
181-
let resultLine = (await runSparqlSelectQueryOnStore(query, store))[0]
182-
if (resultLine) dataPoint.label = resultLine.label
183-
}
184-
185157
let optionals = askUserForDataPoints.filter(missing => missing.optional)
186158
let blockers = askUserForDataPoints.filter(missing => !missing.optional)
187159

@@ -196,8 +168,7 @@ export async function validateOne(userProfile, requirementProfile, datafieldsStr
196168
result: ValidationResult.UNDETERMINABLE,
197169
violations: [],
198170
missingUserInput: askUserForDataPoints,
199-
inMemoryMaterializedTriples: [],
200-
metadata: rpMetadata
171+
inMemoryMaterializedTriples: []
201172
}
202173
}
203174

@@ -241,7 +212,7 @@ export async function validateOne(userProfile, requirementProfile, datafieldsStr
241212
?shape a sh:NodeShape .
242213
FILTER(?shape = ff:MainPersonShape) .
243214
?shape sh:property ?propertyShape .
244-
?propertyShape sh:path <${optional.predicate}> .
215+
?propertyShape sh:path <${optional.dfUri}> .
245216
?propertyShape ?pred ?obj .
246217
}
247218
` // can this query be simplified?
@@ -275,8 +246,7 @@ export async function validateOne(userProfile, requirementProfile, datafieldsStr
275246
result: secondReport.conforms ? ValidationResult.ELIGIBLE : ValidationResult.INELIGIBLE,
276247
violations: collectViolations(secondReport, false),
277248
missingUserInput: askUserForDataPoints,
278-
inMemoryMaterializedTriples: materializedTriples,
279-
metadata: rpMetadata
249+
inMemoryMaterializedTriples: materializedTriples
280250
}
281251
}
282252

‎src/utils.js

+48-18
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import { QueryEngine } from "@comunica/query-sparql-rdfjs"
66

77
export async function rdfStringsToStore(rdfStrings) {
88
let store = new Store()
9-
for (let rdfString of rdfStrings) {
10-
await addRdfStringToStore(rdfString, store)
9+
for (let str of rdfStrings) {
10+
await addRdfStringToStore(str, store)
1111
}
1212
return store
1313
}
@@ -114,29 +114,59 @@ export async function runSparqlDeleteQueryOnStore(query, store) {
114114
return await queryEngine.queryVoid(query, { sources: [ store ] })
115115
}
116116

117-
export async function extractRequirementProfilesMetadata(shaclFileContents) {
118-
let store = await rdfStringsToStore(shaclFileContents)
119-
return await extractRequirementProfilesMetadataFromStore(store)
117+
export async function extractRpUriFromRpString(requirementProfileStr) {
118+
let store = await rdfStringToStore(requirementProfileStr)
119+
let query = `
120+
PREFIX ff: <https://foerderfunke.org/default#>
121+
SELECT * WHERE {
122+
?rpUri a ff:RequirementProfile .
123+
}`
124+
let rows = await runSparqlSelectQueryOnStore(query, store)
125+
return rows[0].rpUri
120126
}
121127

122-
export async function extractRequirementProfilesMetadataFromStore(store) {
128+
export async function extractRequirementProfilesMetadata(requirementProfileStrings) {
129+
let store = await rdfStringsToStore(requirementProfileStrings)
123130
let query = `
124-
PREFIX ff: <https://foerderfunke.org/default#>
125-
SELECT * WHERE {
126-
?id a ff:RequirementProfile .
127-
?id ff:title ?title .
128-
OPTIONAL { ?id ff:category ?category } .
129-
}`
130-
let rqMetadata = {}
131+
PREFIX ff: <https://foerderfunke.org/default#>
132+
SELECT * WHERE {
133+
?rpUri a ff:RequirementProfile .
134+
OPTIONAL { ?rpUri ff:title ?title } .
135+
OPTIONAL { ?rpUri ff:category ?category } .
136+
}`
137+
let metadata = {}
131138
let rows = await runSparqlSelectQueryOnStore(query, store)
132139
for (let row of rows) {
133-
if (!rqMetadata[row.id]) {
134-
rqMetadata[row.id] = {
135-
title: row.title,
140+
if (!metadata[row.rpUri]) {
141+
metadata[row.rpUri] = {
142+
uri: row.rpUri,
143+
title: row.title ?? "",
136144
categories: []
137145
}
138146
}
139-
if (row.category) rqMetadata[row.id].categories.push(row.category)
147+
if (row.category) metadata[row.rpUri].categories.push(row.category)
148+
}
149+
return metadata
150+
}
151+
152+
export async function extractDatafieldsMetadata(datafieldsStr) {
153+
let store = await rdfStringToStore(datafieldsStr)
154+
let query = `
155+
PREFIX ff: <https://foerderfunke.org/default#>
156+
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
157+
SELECT * WHERE {
158+
?dfUri a ff:DataField .
159+
OPTIONAL { ?dfUri rdfs:label ?label } .
160+
OPTIONAL { ?dfUri rdfs:comment ?comment } .
161+
}`
162+
let metadata = {}
163+
let rows = await runSparqlSelectQueryOnStore(query, store)
164+
for (let row of rows) {
165+
metadata[row.dfUri] = {
166+
uri: row.dfUri,
167+
label: row.label ?? "",
168+
comment: row.comment ?? ""
169+
}
140170
}
141-
return rqMetadata
171+
return metadata
142172
}

0 commit comments

Comments
 (0)