Skip to content

Commit c9f107a

Browse files
grahamegrievegrahamegrieve
authored
Merge pull request #1256 from jmandel/perf/db-batch-inserts
perf: Batch SQLite inserts in DBBuilder — 85s → 0.1s in htmlOutputs phase
2 parents af55546 + b0697ad commit c9f107a

1 file changed

Lines changed: 126 additions & 66 deletions

File tree

  • org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers

‎org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/DBBuilder.java‎

Lines changed: 126 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,18 @@ private int getColByName(ResultSetMetaData rsmd, String source) throws SQLExcept
267267

268268
private long cumulativeTime;
269269

270+
// Deferred DB operations: collected during parallel phases, flushed after
271+
private final java.util.concurrent.ConcurrentLinkedQueue<DeferredExpansion> deferredExpansions = new java.util.concurrent.ConcurrentLinkedQueue<>();
272+
273+
private static class DeferredExpansion {
274+
final ValueSet vs;
275+
final ValueSetExpansionOutcome exp;
276+
DeferredExpansion(ValueSet vs, ValueSetExpansionOutcome exp) {
277+
this.vs = vs;
278+
this.exp = exp;
279+
}
280+
}
281+
270282
private void time(long start) {
271283
cumulativeTime = cumulativeTime + (System.currentTimeMillis() - start);
272284
}
@@ -442,55 +454,75 @@ public void finishResources() {
442454
}
443455

444456
try {
445-
PreparedStatement psql = con.prepareStatement("Insert into Properties (Key, ResourceKey, Code, Uri, Description, Type) "+
446-
"values (?, ?, ?, ?, ?, ?)");
447-
for (CodeSystem cs : codesystems) {
448-
for (PropertyComponent p : cs.getProperty()) {
449-
psql.setInt(1, ++lastPropKey);
450-
psql.setInt(2, ((Integer) cs.getUserData(UserDataNames.db_key)).intValue());
451-
bindString(psql, 3, p.getCode());
452-
bindString(psql, 4, p.getUri());
453-
bindString(psql, 5, p.getDescription());
454-
bindString(psql, 6, p.getType().toCode());
455-
psql.executeUpdate();
456-
p.setUserData(UserDataNames.db_key, lastPropKey);
457+
boolean origAutoCommit = con.getAutoCommit();
458+
try {
459+
con.setAutoCommit(false);
460+
461+
PreparedStatement psql = con.prepareStatement("Insert into Properties (Key, ResourceKey, Code, Uri, Description, Type) "+
462+
"values (?, ?, ?, ?, ?, ?)");
463+
for (CodeSystem cs : codesystems) {
464+
for (PropertyComponent p : cs.getProperty()) {
465+
psql.setInt(1, ++lastPropKey);
466+
psql.setInt(2, ((Integer) cs.getUserData(UserDataNames.db_key)).intValue());
467+
bindString(psql, 3, p.getCode());
468+
bindString(psql, 4, p.getUri());
469+
bindString(psql, 5, p.getDescription());
470+
bindString(psql, 6, p.getType().toCode());
471+
psql.addBatch();
472+
p.setUserData(UserDataNames.db_key, lastPropKey);
473+
}
457474
}
458-
}
459-
psql = con.prepareStatement("Insert into Concepts (Key, ResourceKey, ParentKey, Code, Display, Definition) "+
460-
"values (?, ?, ?, ?, ?, ?)");
461-
for (CodeSystem cs : codesystems) {
462-
addConcepts(cs, cs.getConcept(), psql, 0);
463-
}
464-
psql = con.prepareStatement("Insert into ConceptProperties (Key, ResourceKey, ConceptKey, PropertyKey, Code, Value) "+
465-
"values (?, ?, ?, ?, ?, ?)");
466-
for (CodeSystem cs : codesystems) {
467-
addConceptProperties(cs, cs.getConcept(), psql);
468-
}
469-
psql = con.prepareStatement("Insert into Designations (Key, ResourceKey, ConceptKey, UseSystem, UseCode, Lang, Value) "+
470-
"values (?, ?, ?, ?, ?, ?, ?)");
471-
for (CodeSystem cs : codesystems) {
472-
addConceptDesignations(cs, cs.getConcept(), psql);
473-
}
475+
psql.executeBatch();
474476

475-
psql = con.prepareStatement("Insert into ConceptMappings (Key, ResourceKey, SourceSystem, SourceVersion, SourceCode, Relationship, TargetSystem, TargetVersion, TargetCode) "+
476-
"values (?, ?, ?, ?, ?, ?, ?, ?, ?)");
477-
for (ConceptMap cm : mappings) {
478-
for (ConceptMapGroupComponent grp : cm.getGroup()) {
479-
for (SourceElementComponent src : grp.getElement()) {
480-
for (TargetElementComponent tgt : src.getTarget()) {
481-
psql.setInt(1, ++lastMapKey);
482-
psql.setInt(2, ((Integer) cm.getUserData(UserDataNames.db_key)).intValue());
483-
bindString(psql, 3, grp.getSourceElement().baseUrl());
484-
bindString(psql, 4, grp.getSourceElement().version());
485-
bindString(psql, 5, src.getCode());
486-
bindString(psql, 6, tgt.getRelationshipElement().primitiveValue());
487-
bindString(psql, 7, grp.getTargetElement().baseUrl());
488-
bindString(psql, 8, grp.getTargetElement().version());
489-
bindString(psql, 9, tgt.getCode());
490-
psql.executeUpdate();
477+
psql = con.prepareStatement("Insert into Concepts (Key, ResourceKey, ParentKey, Code, Display, Definition) "+
478+
"values (?, ?, ?, ?, ?, ?)");
479+
for (CodeSystem cs : codesystems) {
480+
addConcepts(cs, cs.getConcept(), psql, 0);
481+
}
482+
psql.executeBatch();
483+
484+
psql = con.prepareStatement("Insert into ConceptProperties (Key, ResourceKey, ConceptKey, PropertyKey, Code, Value) "+
485+
"values (?, ?, ?, ?, ?, ?)");
486+
for (CodeSystem cs : codesystems) {
487+
addConceptProperties(cs, cs.getConcept(), psql);
488+
}
489+
psql.executeBatch();
490+
491+
psql = con.prepareStatement("Insert into Designations (Key, ResourceKey, ConceptKey, UseSystem, UseCode, Lang, Value) "+
492+
"values (?, ?, ?, ?, ?, ?, ?)");
493+
for (CodeSystem cs : codesystems) {
494+
addConceptDesignations(cs, cs.getConcept(), psql);
495+
}
496+
psql.executeBatch();
497+
498+
psql = con.prepareStatement("Insert into ConceptMappings (Key, ResourceKey, SourceSystem, SourceVersion, SourceCode, Relationship, TargetSystem, TargetVersion, TargetCode) "+
499+
"values (?, ?, ?, ?, ?, ?, ?, ?, ?)");
500+
for (ConceptMap cm : mappings) {
501+
for (ConceptMapGroupComponent grp : cm.getGroup()) {
502+
for (SourceElementComponent src : grp.getElement()) {
503+
for (TargetElementComponent tgt : src.getTarget()) {
504+
psql.setInt(1, ++lastMapKey);
505+
psql.setInt(2, ((Integer) cm.getUserData(UserDataNames.db_key)).intValue());
506+
bindString(psql, 3, grp.getSourceElement().baseUrl());
507+
bindString(psql, 4, grp.getSourceElement().version());
508+
bindString(psql, 5, src.getCode());
509+
bindString(psql, 6, tgt.getRelationshipElement().primitiveValue());
510+
bindString(psql, 7, grp.getTargetElement().baseUrl());
511+
bindString(psql, 8, grp.getTargetElement().version());
512+
bindString(psql, 9, tgt.getCode());
513+
psql.addBatch();
514+
}
491515
}
492516
}
493517
}
518+
psql.executeBatch();
519+
520+
con.commit();
521+
} catch (SQLException e) {
522+
try { con.rollback(); } catch (SQLException ignored) {}
523+
throw e;
524+
} finally {
525+
con.setAutoCommit(origAutoCommit);
494526
}
495527
} catch (SQLException e) {
496528
errors.add(e.getMessage());
@@ -502,31 +534,54 @@ public void finishResources() {
502534
}
503535

504536
public synchronized void recordExpansion(ValueSet vs, ValueSetExpansionOutcome exp) throws SQLException {
505-
long start = System.currentTimeMillis();
506-
try {
507-
if (con == null) {
508-
return;
509-
}
510-
if (exp == null || exp.getValueset() == null) {
511-
return;
512-
}
537+
// Queue for deferred execution to avoid SQLite lock contention during parallel phases
538+
deferredExpansions.add(new DeferredExpansion(vs, exp));
539+
}
513540

541+
/** Flush all deferred DB operations. Call after parallel phase completes (single-threaded). */
542+
public synchronized void flushDeferredOperations() throws SQLException {
543+
if (con == null) {
544+
// drain queue even if no connection
545+
while (deferredExpansions.poll() != null) {}
546+
return;
547+
}
548+
boolean origAutoCommit = con.getAutoCommit();
549+
try {
550+
con.setAutoCommit(false);
514551
PreparedStatement psql = con.prepareStatement("Insert into ValueSet_Codes (Key, ResourceKey, ValueSetUri, ValueSetVersion, System, Version, Code, Display) "+
515552
"values (?, ?, ?, ?, ?, ?, ?, ?)");
516-
for (ValueSetExpansionContainsComponent e : exp.getValueset().getExpansion().getContains()) {
517-
addContains(vs, e, psql);
553+
DeferredExpansion de;
554+
int batchCount = 0;
555+
while ((de = deferredExpansions.poll()) != null) {
556+
if (de.exp == null || de.exp.getValueset() == null) {
557+
continue;
558+
}
559+
long start = System.currentTimeMillis();
560+
try {
561+
for (ValueSetExpansionContainsComponent e : de.exp.getValueset().getExpansion().getContains()) {
562+
batchCount = addContainsBatch(de.vs, e, psql, batchCount);
563+
}
564+
} catch (SQLException e) {
565+
errors.add(e.getMessage());
566+
if (debug) {
567+
e.printStackTrace();
568+
}
569+
}
570+
time(start);
518571
}
519-
} catch (SQLException e) {
520-
errors.add(e.getMessage());
521-
if (debug) {
522-
e.printStackTrace();
572+
if (batchCount > 0) {
573+
psql.executeBatch();
523574
}
575+
con.commit();
576+
} catch (SQLException e) {
577+
try { con.rollback(); } catch (SQLException ignored) {}
578+
throw e;
579+
} finally {
580+
con.setAutoCommit(origAutoCommit);
524581
}
525-
time(start);
526582
}
527583

528-
529-
private void addContains(ValueSet vs, ValueSetExpansionContainsComponent e, PreparedStatement psql) throws SQLException {
584+
private int addContainsBatch(ValueSet vs, ValueSetExpansionContainsComponent e, PreparedStatement psql, int batchCount) throws SQLException {
530585
if (vs.hasUserData(UserDataNames.db_key)) {
531586
psql.setInt(1, ++lastVSKey);
532587
psql.setInt(2, ((Integer) vs.getUserData(UserDataNames.db_key)).intValue());
@@ -536,11 +591,16 @@ private void addContains(ValueSet vs, ValueSetExpansionContainsComponent e, Prep
536591
bindString(psql, 6, e.getVersion());
537592
bindString(psql, 7, e.getCode());
538593
bindString(psql, 8, e.getDisplay());
539-
psql.executeUpdate();
594+
psql.addBatch();
595+
batchCount++;
596+
if (batchCount % 5000 == 0) {
597+
psql.executeBatch();
598+
}
540599
for (ValueSetExpansionContainsComponent c : e.getContains()) {
541-
addContains(vs, c, psql);
600+
batchCount = addContainsBatch(vs, c, psql, batchCount);
542601
}
543602
}
603+
return batchCount;
544604
}
545605

546606
private void addConcepts(CodeSystem cs, List<ConceptDefinitionComponent> list, PreparedStatement psql, int parent) throws SQLException {
@@ -556,7 +616,7 @@ private void addConcepts(CodeSystem cs, List<ConceptDefinitionComponent> list, P
556616
bindString(psql, 4, cd.getCode());
557617
bindString(psql, 5, cd.getDisplay());
558618
bindString(psql, 6, cd.getDefinition());
559-
psql.executeUpdate();
619+
psql.addBatch();
560620
cd.setUserData(UserDataNames.db_key, lastConceptKey);
561621
addConcepts(cs, cd.getConcept(), psql, lastConceptKey);
562622
}
@@ -578,7 +638,7 @@ private void addConceptProperties(CodeSystem cs, List<ConceptDefinitionComponent
578638
}
579639
bindString(psql, 5, p.getCode());
580640
bindString(psql, 6, p.getValue() == null ? p.getValue().primitiveValue() : null);
581-
psql.executeUpdate();
641+
psql.addBatch();
582642
p.setUserData(UserDataNames.db_key, lastCPropKey);
583643
}
584644
addConceptProperties(cs, cd.getConcept(), psql);
@@ -602,7 +662,7 @@ private void addConceptDesignations(CodeSystem cs, List<ConceptDefinitionCompone
602662
}
603663
bindString(psql, 6, p.getLanguage());
604664
bindString(psql, 7, p.getValue());
605-
psql.executeUpdate();
665+
psql.addBatch();
606666
p.setUserData(UserDataNames.db_key, lastDesgKey);
607667
}
608668
addConceptDesignations(cs, cd.getConcept(), psql);

0 commit comments

Comments
 (0)