diff --git a/importers/build.gradle b/importers/build.gradle index 896557f681..b6fa1c8963 100644 --- a/importers/build.gradle +++ b/importers/build.gradle @@ -89,6 +89,8 @@ dependencies { shadowJar { archiveBaseName = "xlimporter" archiveClassifier = null // removes `-all` in the filename of the created .jar + duplicatesStrategy = DuplicatesStrategy.INCLUDE + mergeServiceFiles() } jar { diff --git a/importers/src/main/groovy/whelk/importer/DatasetImporter.groovy b/importers/src/main/groovy/whelk/importer/DatasetImporter.groovy index 4251ca2e51..096fee6a4a 100644 --- a/importers/src/main/groovy/whelk/importer/DatasetImporter.groovy +++ b/importers/src/main/groovy/whelk/importer/DatasetImporter.groovy @@ -9,6 +9,7 @@ import whelk.JsonLd import whelk.TargetVocabMapper import whelk.Whelk import whelk.converter.TrigToJsonLdParser +import whelk.converter.JsonLdShapes import whelk.util.DocumentUtil import java.time.Duration @@ -77,9 +78,20 @@ class DatasetImporter { DatasetImporter(Whelk whelk, String datasetUri, Map flags=[:], Object descriptions=null) { this.whelk = whelk this.datasetUri = datasetUri + if (datasetUri != null) { + log.info("Initialized DatasetImporter for ${datasetUri}") + } + log.info("Using system context: ${whelk.systemContextUri}") if (whelk.systemContextUri) { contextDocData = getDocByMainEntityId(whelk.systemContextUri)?.data + if (contextDocData.containsKey(CONTEXT)) { + def ctx = contextDocData.get(CONTEXT) + if (ctx instanceof Map) log.info("Context size: ${ctx.size()}") + } else { + log.warn("Context missing ${CONTEXT}") + } } + if (descriptions != null) { Map datasetDesc = descriptions instanceof Map ? (Map) descriptions : loadData((String) descriptions) givenDsData = (Map) findInData(datasetDesc, datasetUri) @@ -96,13 +108,14 @@ class DatasetImporter { } static void loadDescribedDatasets(Whelk whelk, String datasetDescPath, String sourceBaseDir, Set onlyDatasets=null, Map flags=[:]) { + log.info("Loading datasets described in: ${datasetDescPath}") var dsImp = new DatasetImporter(whelk, null) var datasets = (Map) new File(datasetDescPath).withInputStream { dsImp.contextDocData ? dsImp.loadTurtleAsSystemShaped(it) : loadSelfCompactedTurtle(it) } for (Map item : (List) datasets[GRAPH] ?: asList(datasets)) { if (onlyDatasets && item[ID] !in onlyDatasets) { - System.err.println("Skipping dataset: ${item[ID]}") + log.info("Skipping dataset: ${item[ID]}") continue } if (item[TYPE] == 'Dataset' && 'sourceData' in item) { @@ -140,7 +153,7 @@ class DatasetImporter { private void doImportDataset(String sourceUrl) { long startTime = System.nanoTime() - System.err.println("Importing from: ${sourceUrl}") + log.info("Importing from: ${sourceUrl}") Set idsInInput = [] @@ -185,7 +198,7 @@ class DatasetImporter { countWriteAndMaybeFlushIndexing() if ( lineCount % 100 == 0 ) { - System.err.println("Processed " + lineCount + " input records. " + createdCount + " created, " + + log.info("Processed " + lineCount + " input records. " + createdCount + " created, " + updatedCount + " updated, " + (lineCount-createdCount-updatedCount) + " already up to date.") } ++lineCount @@ -205,11 +218,11 @@ class DatasetImporter { Duration elapsedTime = Duration.ofNanos(System.nanoTime() - startTime) String elapsed = String.format("%02dh%02dm%02ds", elapsedTime.toHours(), elapsedTime.toMinutesPart(), elapsedTime.toSecondsPart()) - System.err.println("Created: " + createdCount +" new,\n" + - "updated: " + updatedCount + " existing and\n" + - "deleted: " + deletedCount + " old records (should have been: " + (deletedCount + needsRetry.size()) + "),\n" + - "out of the: " + idsInInput.size() + " records in dataset: \"" + dsInfo.uri + "\".\n" + - "Dataset now in sync in ${elapsed}.") + log.info("Created: " + createdCount +" new,\n" + + "\tupdated: " + updatedCount + " existing and\n" + + "\tdeleted: " + deletedCount + " old records (should have been: " + (deletedCount + needsRetry.size()) + "),\n" + + "\tout of the: " + idsInInput.size() + " records in dataset: \"" + dsInfo.uri + "\".\n" + + "\tDataset now in sync in ${elapsed}.") } void dropDataset() { @@ -223,7 +236,7 @@ class DatasetImporter { } finally { whelk.endDeferredIndexing() } - System.err.println("Deleted dataset ${dsInfo.uri} with ${deletedCount} existing records") + log.info("Deleted dataset ${dsInfo.uri} with ${deletedCount} existing records") } private void processDataset(String sourceUrl, Closure processItem) { @@ -251,16 +264,16 @@ class DatasetImporter { Map selfDescribedDsData = findInData(data, datasetUri) String dsId = null if (selfDescribedDsData != null) { - System.err.println("Using self-described dataset description") + log.info("Using self-described dataset description") setDatasetInfo(datasetUri, data) } else if (givenDsData != null) { - System.err.println("Using given dataset description") + log.info("Using given dataset description") setDatasetInfo(datasetUri, givenDsData) dsRecord = completeRecord(givenDsData, JsonLd.SYSTEM_RECORD_TYPE) createOrUpdateDocument(dsRecord) dsId = dsRecord.getShortId() } else if (useExistingDatasetDescription) { - System.err.println("Using existing dataset description") + log.info("Using existing dataset description") lookupDatasetInfo(datasetUri) } return dsId @@ -272,7 +285,7 @@ class DatasetImporter { throw new RuntimeException("Provided dataset ${givenData[ID]} does not match: ${datasetUri}") } dsInfo = new DatasetInfo(dsData) - System.err.println("Using new dataset: ${dsInfo.uri}") + log.info("Using new dataset: ${dsInfo.uri}") } protected void lookupDatasetInfo(String datasetUri) { @@ -283,7 +296,7 @@ class DatasetImporter { Map datasetData = ((List) datasetRecord.data[GRAPH])[1] assert datasetData[ID] == datasetUri dsInfo = new DatasetInfo(datasetData) - System.err.println("Using already defined dataset: ${dsInfo.uri}") + log.info("Using already defined dataset: ${dsInfo.uri}") } protected Document completeRecord(Map data, String recordType, boolean remap = false) { @@ -372,9 +385,14 @@ class DatasetImporter { } } + /** + * This method is only intended for initial whelk bootstrapping, + * to initially load systemContext (contextDocData). + */ private static Map loadSelfCompactedTurtle(InputStream ins) { // Assuming that the Turtle *shape* follows a hard-coded system context! - Map data = (Map) TrigToJsonLdParser.parse(ins) + log.warn("Using loadSelfCompactedTurtle should only happen during setup of fresh whelk installation") + Map data = (Map) TrigToJsonLdParser.parseRaw(ins) if (CONTEXT in data) { Map ctx = [:] ctx.putAll((Map) data[CONTEXT]) @@ -384,18 +402,27 @@ class DatasetImporter { ctx['xsd'] = XSD_NS // Assumes VOCAB + created in source actually means this! ctx['created'] = [(TYPE): 'xsd:dateTime'] - data = (Map) TrigToJsonLdParser.compact(data, [(CONTEXT): ctx]) + data = (Map) JsonLdShapes.reCompact(data, [(CONTEXT): ctx]) } return data } private Map loadTurtleAsSystemShaped(InputStream ins) { assert contextDocData - Map data = TrigToJsonLdParser.parse(ins) + Map data = TrigToJsonLdParser.parseRaw(ins) + if (checkedSystemShaped(data, whelk.jsonld.vocabId)) { + return (Map) JsonLdShapes.reCompact(data, contextDocData) + } else { + log.info("Applying target vocabulary map") + return (Map) getTvm().applyTargetVocabularyMap(whelk.systemContextUri, contextDocData, data) + } + } + + private static boolean checkedSystemShaped(Map data, String vocabId) { if (data[CONTEXT] instanceof Map) { Map ctx = (Map) data[CONTEXT] int expectedSize = 0 - if (ctx[VOCAB] == whelk.jsonld.vocabId) { + if (ctx[VOCAB] == vocabId) { expectedSize++ if (ctx.containsKey(BASE)) { expectedSize++ @@ -405,14 +432,15 @@ class DatasetImporter { } } if (ctx.size() == expectedSize) { - // Forces plain string uri values to be taken as datatyped + // TODO: remove this hack when definitions consistently use `:uri ""^^xsd:anyURI`! + // Force plain string uri value to be expanded as datatyped: if ('uri' !in ctx) { ctx['uri'] = [(TYPE): 'xsd:anyURI'] } - return (Map) TrigToJsonLdParser.compact(data, contextDocData) + return true } } - return (Map) getTvm().applyTargetVocabularyMap(whelk.systemContextUri, contextDocData, data) + return false } private Document getDocByMainEntityId(String id) { @@ -477,7 +505,7 @@ class DatasetImporter { } else { deletedCount++ if (deletedCount % 50 == 0) { - System.err.println("Cleaning up: " + deletedCount + " records deleted (they are no longer in the dataset).") + log.info("Cleaning up: " + deletedCount + " records deleted (they are no longer in the dataset).") } } } diff --git a/importers/src/main/groovy/whelk/importer/ImporterMain.groovy b/importers/src/main/groovy/whelk/importer/ImporterMain.groovy index 3eb734c09e..f9df255944 100644 --- a/importers/src/main/groovy/whelk/importer/ImporterMain.groovy +++ b/importers/src/main/groovy/whelk/importer/ImporterMain.groovy @@ -3,8 +3,17 @@ package whelk.importer import java.lang.annotation.* import java.util.concurrent.ExecutorService import java.util.zip.GZIPOutputStream + +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes + import groovy.cli.picocli.CliBuilder import groovy.util.logging.Slf4j as Log + import org.apache.commons.io.output.CountingOutputStream import org.apache.commons.io.FilenameUtils @@ -12,9 +21,11 @@ import whelk.Document import whelk.Whelk import whelk.component.PostgreSQLComponent import whelk.converter.JsonLdToTrigSerializer +import whelk.converter.RdfReader import whelk.filter.LinkFinder import whelk.reindexer.CardRefresher import whelk.reindexer.ElasticReindexer +import whelk.util.Jackson import whelk.util.PropertyLoader @Log @@ -30,6 +41,52 @@ class ImporterMain { return Whelk.createLoadedSearchWhelk(props) } + @Command(args='SOURCE_URL [SUFFIXES]') + void rdfDirToJsonLines(String sourceDir, String suffixes='rdf,ttl,jsonld') { + var whelk = Whelk.createLoadedCoreWhelk(props) + + var systemContextUri = whelk.systemContextUri + var baseUri = whelk.baseUri.toString() + System.err.println "Whelk system context URI: $systemContextUri; base URI: $baseUri" + + var context = whelk.storage.loadDocumentByMainId(systemContextUri).data + + var sourceDirPath = Paths.get(sourceDir) + var matcher = sourceDirPath.getFileSystem().getPathMatcher("glob:**/*.{$suffixes}") + + var printStream = System.out + + Files.walkFileTree(sourceDirPath, new SimpleFileVisitor() { + @Override + FileVisitResult visitFile(Path path, BasicFileAttributes attrs) { + if (!matcher.matches(path)) { + return FileVisitResult.CONTINUE + } + + var rdfSourcePath = path.toString() + var data = new File(rdfSourcePath).withInputStream { + RdfReader.readRdf(it, rdfSourcePath, context, systemContextUri, baseUri) + } + + if ('@graph' in data) { + // Ensure LDDB-expected order of things: + Map record = data['@graph'].find { it -> it['@type'] == 'Record' } + if (record) { + // assumes correctly structured record + def mainId = record['mainEntity']['@id'] + Map mainEntity = data['@graph'].find { it -> it['@id'] == mainId } + List rest = data['@graph'].findAll { it -> !it.is(record) && !it.is(mainEntity) } + data = ['@graph': [record, mainEntity] + rest] + } + } + + printStream.println(Jackson.mapper.writeValueAsString(data)) + + return FileVisitResult.CONTINUE + } + }) + } + @Command(args='SOURCE_URL DATASET_URI [DATASET_DESCRIPTION_FILE]', flags='--skip-index --replace-main-ids --force-delete --skip-dependers --allow-id-removal') void dataset(Map flags, String sourceUrl, String datasetUri, String datasetDescPath=null) { diff --git a/importers/src/test/groovy/whelk/importer/DatasetImporterSpec.groovy b/importers/src/test/groovy/whelk/importer/DatasetImporterSpec.groovy new file mode 100644 index 0000000000..c9c6c538cb --- /dev/null +++ b/importers/src/test/groovy/whelk/importer/DatasetImporterSpec.groovy @@ -0,0 +1,72 @@ +package whelk.importer + +import spock.lang.Specification + +import whelk.converter.TrigToJsonLdParser + +class DatasetImporterSpec extends Specification { + + def "load self compacted turtle"() { + given: + var s = """ + prefix : + a :Record . + """ + var data = DatasetImporter.loadSelfCompactedTurtle(new ByteArrayInputStream(s.getBytes('utf-8'))) + + expect: + data == [ + "@id": "x", + "@type": "Record" + ] + } + + def "check system-shaped turtle"() { + when: + var s = """ + prefix : + prefix xsd: + + a :Dataset ; + :uri "ds/1" ; # NOTE: plain string form (notice difference in JSON-LD) + :created "2026-10-01T01:00:00Z"^^xsd:dateTime . + a :Dataset ; + :uri "ds/2"^^xsd:anyURI ; + :created "2026-10-01T02:00:00Z"^^xsd:dateTime . + """ + var data = TrigToJsonLdParser.parse(new ByteArrayInputStream(s.getBytes('utf-8'))) + + then: + data == [ + "@context": [ + "@vocab": "https://id.kb.se/vocab/", + "xsd": "http://www.w3.org/2001/XMLSchema#" + ], + "@graph": [ + [ + "@id": "ds/1", + "@type": "Dataset", + "uri": "ds/1" , + "created": ["@type": "xsd:dateTime", "@value": "2026-10-01T01:00:00Z"] + ], + [ + "@id": "ds/2", + "@type": "Dataset", + "uri": ["@type": "xsd:anyURI", "@value": "ds/2"], + "created": ["@type": "xsd:dateTime", "@value": "2026-10-01T02:00:00Z"] + ] + ] + ] + + and: + DatasetImporter.checkedSystemShaped(data, "https://id.kb.se/vocab/") + + and: + data["@context"] == [ + "@vocab": "https://id.kb.se/vocab/", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "uri": ["@type": "xsd:anyURI"] + ] + } + +} diff --git a/whelk-core/src/main/groovy/whelk/converter/JsonLdShapes.java b/whelk-core/src/main/groovy/whelk/converter/JsonLdShapes.java new file mode 100644 index 0000000000..43d679b4c1 --- /dev/null +++ b/whelk-core/src/main/groovy/whelk/converter/JsonLdShapes.java @@ -0,0 +1,42 @@ +package whelk.converter; + +import java.util.Map; + +import trld.jsonld.Compaction; +import trld.jsonld.Expansion; +import trld.jsonld.Flattening; + +public class JsonLdShapes { + public static Object reCompact(Object data, Map context) { + return reCompact(data, context, null); + } + + public static Object reCompact(Object data, Map context, String baseIri) { + var expanded = expand(data, baseIri); + return Compaction.compact(context, expanded); + } + + public static Object reCompactWithEmbeddedBlanks(Object data, Map context) { + return reCompactWithEmbeddedBlanks(data, context, null); + } + + public static Object reCompactWithEmbeddedBlanks(Object data, Map context, String baseIri) { + var expanded = expand(data, baseIri); + var flattened = flatten(expanded); + var compacted = Compaction.compact(context, flattened, baseIri); + EmbedBlanks.embedBlanks(compacted); + return compacted; + } + + protected static Object expand(Object data) { + return expand(data, null); + } + + protected static Object expand(Object data, String baseIri) { + return Expansion.expand(data, baseIri); + } + + protected static Object flatten(Object data) { + return Flattening.flatten(data); + } +} diff --git a/whelk-core/src/main/groovy/whelk/converter/RdfReader.java b/whelk-core/src/main/groovy/whelk/converter/RdfReader.java index bdf129799f..c722667a2a 100644 --- a/whelk-core/src/main/groovy/whelk/converter/RdfReader.java +++ b/whelk-core/src/main/groovy/whelk/converter/RdfReader.java @@ -12,13 +12,21 @@ public class RdfReader { static Map readRdf(InputStream bis, String rdfSourcePath, Map context) throws IOException { + return readRdf(bis, rdfSourcePath, context, null); + } + + static Map readRdf(InputStream bis, String rdfSourcePath, Map context, String contextUri) throws IOException { + return readRdf(bis, rdfSourcePath, context, contextUri, null); + } + + static Map readRdf(InputStream bis, String rdfSourcePath, Map context, String contextUri, String baseIri) throws IOException { Map data; if (rdfSourcePath.endsWith(".ttl")) { - data = readTurtle(bis, rdfSourcePath, context); + data = readTurtle(bis, rdfSourcePath, context, baseIri); } else if (rdfSourcePath.endsWith(".rdf")) { - data = readRdfXml(bis, rdfSourcePath, context); + data = readRdfXml(bis, rdfSourcePath, context, baseIri); } else if (rdfSourcePath.endsWith(".jsonld")) { - data = readJsonLd(bis, rdfSourcePath, context); + data = readJsonLd(bis, rdfSourcePath, context, contextUri, baseIri); } else { throw new WhelkRuntimeException("Unknown RDF format for ${rdfSourcePath}"); } @@ -37,17 +45,20 @@ static Map readRdf(InputStream bis, String rdfSourcePath, Map context) throws IO } } - static Map readTurtle(InputStream bis, String rdfSourcePath, Map context) throws IOException { - return TrigToJsonLdParser.parse(bis, context); + private static Map readTurtle(InputStream bis, String rdfSourcePath, Map context, String baseIri) throws IOException { + return TrigToJsonLdParser.parse(bis, context, baseIri); } - static Map readRdfXml(InputStream bis, String rdfSourcePath, Map context) throws IOException { - return (Map) RdfXmlToJsonLdParser.parse(bis, context); + private static Map readRdfXml(InputStream bis, String rdfSourcePath, Map context, String baseIri) throws IOException { + return (Map) RdfXmlToJsonLdParser.parse(bis, context, baseIri); } - static Map readJsonLd(InputStream bis, String rdfSourcePath, Map context) throws IOException { + private static Map readJsonLd(InputStream bis, String rdfSourcePath, Map context, String contextUri, String baseIri) throws IOException { Map data = mapper.readValue(bis, Map.class); - return (Map) TrigToJsonLdParser.compact(data, context); + if (contextUri != null && contextUri.equals(data.get(CONTEXT_KEY))) { + data.put(CONTEXT_KEY, context.get(CONTEXT_KEY)); + } + return (Map) JsonLdShapes.reCompactWithEmbeddedBlanks(data, context, baseIri); } } diff --git a/whelk-core/src/main/groovy/whelk/converter/RdfXmlToJsonLdParser.java b/whelk-core/src/main/groovy/whelk/converter/RdfXmlToJsonLdParser.java index 798227f382..255e0ad508 100644 --- a/whelk-core/src/main/groovy/whelk/converter/RdfXmlToJsonLdParser.java +++ b/whelk-core/src/main/groovy/whelk/converter/RdfXmlToJsonLdParser.java @@ -14,15 +14,18 @@ public static Object parse(InputStream inStream) throws IOException { } public static Object parse(InputStream inStream, Map context) throws IOException { + return parse(inStream, context, null); + } + + public static Object parse(InputStream inStream, Map context, String baseIri) throws IOException { var model = ModelFactory.createDefaultModel(); - model.read(inStream, "RDF/XML"); + model.read(inStream, baseIri, "RDF/XML"); var baos = new ByteArrayOutputStream(); try (var outSteam = new OutputStreamWriter(baos)) { model.write(outSteam, "JSONLD"); } var inData = mapper.readValue(baos.toString("UTF-8"), Map.class); - var data = TrigToJsonLdParser.compact(inData, context); - EmbedBlanks.embedBlanks(data); + var data = JsonLdShapes.reCompactWithEmbeddedBlanks(inData, context, baseIri); return data; } diff --git a/whelk-core/src/main/groovy/whelk/converter/TrigToJsonLdParser.groovy b/whelk-core/src/main/groovy/whelk/converter/TrigToJsonLdParser.groovy deleted file mode 100644 index 2b760c62db..0000000000 --- a/whelk-core/src/main/groovy/whelk/converter/TrigToJsonLdParser.groovy +++ /dev/null @@ -1,29 +0,0 @@ -package whelk.converter - -import groovy.transform.CompileStatic - -import trld.jsonld.Compaction -import trld.jsonld.Expansion -import trld.platform.Output -import trld.platform.Input -import trld.trig.Parser - -@CompileStatic -class TrigToJsonLdParser { - public static Map parse(InputStream inStream) { - return parse(inStream, (Map) null); - } - - public static Map parse(InputStream inStream, Map context) throws IOException { - Map data = (Map) Parser.parse(new Input(inStream)) - return (Map) compact(data, context) - } - - static Object expand(Object data, String baseIri=null) { - Expansion.expand(data, baseIri) - } - - static Object compact(Object data, Map context, String baseIri=null) { - return Compaction.compact(context, expand(data, baseIri)) - } -} diff --git a/whelk-core/src/main/groovy/whelk/converter/TrigToJsonLdParser.java b/whelk-core/src/main/groovy/whelk/converter/TrigToJsonLdParser.java new file mode 100644 index 0000000000..64c2921645 --- /dev/null +++ b/whelk-core/src/main/groovy/whelk/converter/TrigToJsonLdParser.java @@ -0,0 +1,31 @@ +package whelk.converter; + +import java.io.InputStream; +import java.io.IOException; +import java.util.Map; + +import trld.platform.Input; +import trld.trig.Parser; + +public class TrigToJsonLdParser { + public static Map parse(InputStream inStream) throws IOException { + return parse(inStream, (Map) null); + } + + public static Map parse(InputStream inStream, Map context) throws IOException { + return parse(inStream, context, null); + } + + public static Map parse(InputStream inStream, Map context, String baseIri) throws IOException { + var data = parseRaw(inStream); + if (context != null) { + return (Map) JsonLdShapes.reCompact(data, context, baseIri); + } else { + return data; + } + } + + public static Map parseRaw(InputStream inStream) throws IOException { + return (Map) Parser.parse(new Input(inStream)); + } +} diff --git a/whelk-core/src/main/groovy/whelk/converter/XmlIterParser.java b/whelk-core/src/main/groovy/whelk/converter/XmlIterParser.java new file mode 100644 index 0000000000..b1995c32cf --- /dev/null +++ b/whelk-core/src/main/groovy/whelk/converter/XmlIterParser.java @@ -0,0 +1,141 @@ +package whelk.converter; + +import javax.xml.stream.XMLEventFactory; +import javax.xml.stream.XMLEventReader; +import javax.xml.stream.XMLEventWriter; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.events.Namespace; +import javax.xml.stream.events.StartElement; +import java.io.InputStream; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +public class XmlIterParser implements Iterator, AutoCloseable { + + private final XMLEventReader reader; + private final XMLOutputFactory outputFactory; + private final XMLEventFactory eventFactory; + + private final List ancestorNamespaces = new ArrayList<>(); + private StartElement childStart = null; + + public XmlIterParser(InputStream is) throws Exception { + var inputFactory = XMLInputFactory.newInstance(); + // Enable DTD entity support (fairly common in RDF/XML) + inputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, true); + // But no external (insecure!) + inputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + + this.reader = inputFactory.createXMLEventReader(is); + this.outputFactory = XMLOutputFactory.newInstance(); + this.eventFactory = XMLEventFactory.newInstance(); + } + + @Override + public boolean hasNext() { + if (childStart == null) { + try { + while (reader.hasNext()) { + var event = reader.nextEvent(); + // Ignore root element + if (event.isStartElement()) { + reader.nextEvent(); + var start = event.asStartElement(); + // Collect active namespaces + var nsIter = start.getNamespaces(); + while (nsIter.hasNext()) { + ancestorNamespaces.add((Namespace) nsIter.next()); + } + break; + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + if (!reader.hasNext()) { + return false; + } + + try { + while (reader.hasNext()) { + var event = reader.peek(); + if (event.isStartElement()) { + childStart = event.asStartElement(); + return true; + } + reader.nextEvent(); // Drop intermediate whitespace + } + return false; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private StartElement completeChildStart(StartElement childStart) { + var allActiveNamespaces = new ArrayList<>(ancestorNamespaces); + var childNsIter = childStart.getNamespaces(); + while (childNsIter.hasNext()) { + var ns = (Namespace) childNsIter.next(); + // Override ancestor duplicates with local declarations + allActiveNamespaces.removeIf(n -> n.getPrefix().equals(ns.getPrefix())); + allActiveNamespaces.add(ns); + } + + return eventFactory.createStartElement( + childStart.getName(), + childStart.getAttributes(), + allActiveNamespaces.iterator() + ); + } + + @Override + public String next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + var sw = new StringWriter(); + XMLEventWriter writer; + try { + writer = outputFactory.createXMLEventWriter(sw); + } catch (Exception e) { + throw new RuntimeException(e); + } + + if (childStart == null) { + throw new NoSuchElementException(); + } + + try { + reader.nextEvent(); // consume + childStart = completeChildStart(childStart); + writer.add(childStart); + + var depth = 1; + while (reader.hasNext() && depth > 0) { + var event = reader.nextEvent(); + writer.add(event); + + if (event.isStartElement()) depth++; + if (event.isEndElement()) depth--; + } + + writer.close(); + return sw.toString(); + + } catch (Exception e) { + throw new RuntimeException("Collecting inner XML failed", e); + } + } + + @Override + public void close() throws Exception { + if (reader != null) reader.close(); + } +} diff --git a/whelktool/scripts/examples/create-from-rdf-directory.groovy b/whelktool/scripts/examples/create-from-rdf-directory.groovy new file mode 100644 index 0000000000..cea127051e --- /dev/null +++ b/whelktool/scripts/examples/create-from-rdf-directory.groovy @@ -0,0 +1,49 @@ +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes + +import whelk.Whelk +import whelk.datatool.DocumentItem +import static whelk.converter.RdfReader.readRdf + +Map loadRdf(Whelk whelk, String rdfSourcePath) { + var context = whelk.storage.loadDocumentByMainId(whelk.systemContextUri).data + return new File(rdfSourcePath).withInputStream { + readRdf(it, rdfSourcePath, context, whelk.systemContextUri, whelk.baseUri.toString()) + } +} + +DocumentItem saveRecord(Map data) { + Map record = data[GRAPH].find { it -> it[TYPE] == 'Record' } + assert record + def mainId = record['mainEntity'][ID] + assert mainId.endsWith('#it') + Map mainEntity = data[GRAPH].find { it -> it[ID] == mainId } + assert mainEntity + List rest = data[GRAPH].findAll { it -> !it.is(record) && !it.is(mainEntity) } + + return create(['@graph': [record, mainEntity] + rest]) +} + +var rdfDirPath = Paths.get(System.getProperty("rdfdir")) +var whelk = getWhelk() + +var matcher = rdfDirPath.getFileSystem().getPathMatcher('glob:**/*.{rdf,ttl,jsonld}') + +Files.walkFileTree(rdfDirPath, new SimpleFileVisitor() { + @Override + FileVisitResult visitFile(Path path, BasicFileAttributes attrs) { + if (!matcher.matches(path)) { + return FileVisitResult.CONTINUE + } + var data = loadRdf(whelk, path.toString()) + var docItem = saveRecord(data) + selectFromIterable([docItem], { newItem -> + newItem.scheduleSave() + }) + return FileVisitResult.CONTINUE + } +}) diff --git a/whelktool/scripts/examples/create-from-rdf.groovy b/whelktool/scripts/examples/create-from-rdf.groovy index f36a5dcc43..e83bcf8896 100644 --- a/whelktool/scripts/examples/create-from-rdf.groovy +++ b/whelktool/scripts/examples/create-from-rdf.groovy @@ -3,8 +3,10 @@ import whelk.datatool.DocumentItem import static whelk.converter.RdfReader.readRdf List loadDescriptions(Whelk whelk, String rdfSourcePath) { - var context = whelk.storage.loadDocumentByMainId(whelk.systemContextUri, null).data - Map data = new File(rdfSourcePath).withInputStream { readRdf(it, rdfSourcePath, context) } + var context = whelk.storage.loadDocumentByMainId(whelk.systemContextUri).data + Map data = new File(rdfSourcePath).withInputStream { + readRdf(it, rdfSourcePath, context, whelk.systemContextUri) + } return data[GRAPH] } @@ -25,7 +27,7 @@ List newDocs = loadDescriptions(getWhelk(), rdfSourcePath).collect { Map ma } var data = ['@graph': [record, mainEntity]] - create(data) + return create(data) } selectFromIterable(newDocs, { newItem -> diff --git a/whelktool/scripts/examples/create-from-rdfxml-stream.groovy b/whelktool/scripts/examples/create-from-rdfxml-stream.groovy new file mode 100644 index 0000000000..9f4f631e0a --- /dev/null +++ b/whelktool/scripts/examples/create-from-rdfxml-stream.groovy @@ -0,0 +1,39 @@ +import java.util.stream.StreamSupport + +import whelk.converter.XmlIterParser +import whelk.converter.RdfXmlToJsonLdParser + +import whelk.Whelk +import whelk.datatool.DocumentItem +import static whelk.converter.RdfReader.readRdf + + +DocumentItem saveRecord(Map data) { + Map record = data[GRAPH].find { it -> it[TYPE] == 'Record' } + assert record + def mainId = record['mainEntity'][ID] + assert mainId.endsWith('#it') + Map mainEntity = data[GRAPH].find { it -> it[ID] == mainId } + assert mainEntity + + return create(['@graph': [record, mainEntity]]) +} + +var rdfSourcePath = System.getProperty("rdfdata") +var file = new File(rdfSourcePath) + +var whelk = getWhelk() +var context = whelk.storage.loadDocumentByMainId(whelk.systemContextUri).data + +file.withInputStream { + var iterParser = new XmlIterParser(it) + + for (var xml in iterParser) { + var bis = new ByteArrayInputStream(xml.getBytes('UTF-8')) + var data = RdfXmlToJsonLdParser.parse(bis, context) + var docItem = saveRecord(data) + selectFromIterable([docItem], { newItem -> + newItem.scheduleSave() + }) + } +} diff --git a/whelktool/scripts/examples/datasets/libris-example-dataset.rdf b/whelktool/scripts/examples/datasets/libris-example-dataset.rdf new file mode 100644 index 0000000000..a83f25a922 --- /dev/null +++ b/whelktool/scripts/examples/datasets/libris-example-dataset.rdf @@ -0,0 +1,18 @@ + + +]> + + + + + + + + + + + + + + diff --git a/whelktool/scripts/examples/records/libris-varied-example-record-kbv-context.jsonld b/whelktool/scripts/examples/records/libris-varied-example-record-kbv-context.jsonld new file mode 100644 index 0000000000..02df8f6c50 --- /dev/null +++ b/whelktool/scripts/examples/records/libris-varied-example-record-kbv-context.jsonld @@ -0,0 +1,84 @@ +{ + "@context": "https://id.kb.se/sys/context/kbv", + "@graph": [ + { + "@id": "local001#it", + "@type": "PhysicalResource", + "category": [ + {"@id": "https://id.kb.se/term/rda/Volume"}, + {"@id": "https://id.kb.se/term/saobf/Print"} + ], + "contribution": { + "@type": "Contribution", + "agent": {"@id": "https://libris.kb.se/wt79bh6f2j46dtr#it"}, + "role": {"@id": "https://id.kb.se/relator/illustrator"} + }, + "editionStatement": "14. uppl.", + "editionStatementRemainder": "illustrationer av Tove Jansson", + "extent": { + "@type": "Extent", + "label": "308 s." + }, + "hasDimensions": { + "@type": "Dimensions", + "label": "24 cm" + }, + "hasNote": { + "@type": "Note", + "label": "Första svenska uppl. 1947. Första uppl. i denna version 1962. - Även utg. med titeln: Hompen" + }, + "hasTitle": { + "@type": "Title", + "mainTitle": [ + "Bilbo", + {"@language": "en", "@value": "The Hobbit"} + ], + "mainTitleByLang": {"de": "Der Hobbit"}, + "subtitle": "en hobbits äventyr" + }, + "identifiedBy": { + "@type": "ISBN", + "acquisitionTerms": "202:00", + "qualifier": "INB.", + "value": "9151827271" + }, + "illustrativeContent": {"@id": "marc:Illustration"}, + "instanceOf": {"@id": "https://libris.kb.se/l3crh1z1j8xdfqmg#it"}, + "manufacture": { + "@type": "Manufacture", + "agent": { + "@type": "Agent", + "label": "Scandbook" + }, + "place": { + "@type": "Place", + "label": "Falun" + } + }, + "publication": { + "@type": "PrimaryPublication", + "agent": { + "@type": "Agent", + "label": "Rabén Prisma" + }, + "country": {"@id": "https://id.kb.se/country/sw"}, + "place": { + "@type": "Place", + "label": "Stockholm" + }, + "year": "1994" + }, + "responsibilityStatement": "J.R.R. Tolkien ; översättning av Britt G. Hallqvist", + "meta": { + "@type": "Record", + "bibliography": {"@id": "https://libris.kb.se/library/NB"}, + "created": "2026-09-29T09:16:44Z", + "descriptionCreator": {"@id": "https://libris.kb.se/library/S"}, + "encodingLevel": [ + "marc:MinimalLevel", + {"@id": "marc:MinimalLevel"} + ] + } + } + ] +}